2024-02-19T21:33:34.779 INFO:root:teuthology version: 0.0.1.dev260+gb1dac55 2024-02-19T21:33:34.779 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_yuriw@teuthology --archive /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773 --verbose --name yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi --description rados/cephadm/mgr-nfs-upgrade/{0-distro/centos_8.stream_container_tools 1-bootstrap/16.2.5 1-start 2-nfs 3-upgrade-with-workload 4-final} -- /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773/orig.config.yaml 2024-02-19T21:33:34.815 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-02-19T21:33:34.894 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773 branch: pacific-release description: rados/cephadm/mgr-nfs-upgrade/{0-distro/centos_8.stream_container_tools 1-bootstrap/16.2.5 1-start 2-nfs 3-upgrade-with-workload 4-final} email: null first_in_suite: false job_id: '7566773' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: pacific-release ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 81bd20d634209c7cb82c18be12b4b5a05643ebf1 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 81bd20d634209c7cb82c18be12b4b5a05643ebf1 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: pacific-release sha1: 81bd20d634209c7cb82c18be12b4b5a05643ebf1 owner: scheduled_yuriw@teuthology priority: 71 repo: https://github.com/ceph/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 264 sha1: 81bd20d634209c7cb82c18be12b4b5a05643ebf1 sleep_before_teardown: 0 subset: 111/120000 suite: rados suite_branch: pacific-release suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_81bd20d634209c7cb82c18be12b4b5a05643ebf1/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: 81bd20d634209c7cb82c18be12b4b5a05643ebf1 targets: smithi131.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAgG0jlF18ihbqFbZzwpagx6+jBYHXzsEII8carzTu7Vw14Qs9Zo+UhC/Py6tfNnPNg7/sVImU3iV+Oq5Wzpj5I= smithi196.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPFvA3EACYn4fifLnkd2R05kSwjMJ7iZwClMGYRZBS3L2isJojg/yFH9RzeBx+ZqIrNJlHzLANMvIdsslznGB9s= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - cephadm: cephadm_branch: v16.2.5 cephadm_git_url: https://github.com/ceph/ceph image: docker.io/ceph/ceph:v16.2.5 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - ceph config get mgr mgr/cephadm/migration_current | grep 5 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: b1dac5519c57c269ea98a22fb7729016a1d0e4d2 timestamp: 2024-02-19_19:25:49 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: yuriw verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-02-19T21:33:34.895 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_81bd20d634209c7cb82c18be12b4b5a05643ebf1/qa; will attempt to use it 2024-02-19T21:33:34.895 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_81bd20d634209c7cb82c18be12b4b5a05643ebf1/qa/tasks 2024-02-19T21:33:34.895 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-02-19T21:33:34.897 INFO:teuthology.task.internal:Checking packages... 2024-02-19T21:33:34.919 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '81bd20d634209c7cb82c18be12b4b5a05643ebf1' 2024-02-19T21:33:34.919 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-02-19T21:33:34.920 INFO:teuthology.packaging:ref: None 2024-02-19T21:33:34.920 INFO:teuthology.packaging:tag: None 2024-02-19T21:33:34.920 INFO:teuthology.packaging:branch: pacific-release 2024-02-19T21:33:34.920 INFO:teuthology.packaging:sha1: 81bd20d634209c7cb82c18be12b4b5a05643ebf1 2024-02-19T21:33:34.920 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific-release 2024-02-19T21:33:35.065 INFO:teuthology.task.internal:Found packages for ceph version 16.2.14-548.g81bd20d6 2024-02-19T21:33:35.066 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-02-19T21:33:35.073 INFO:teuthology.task.internal:no buildpackages task found 2024-02-19T21:33:35.073 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-02-19T21:33:35.093 INFO:teuthology.task.internal:Saving configuration 2024-02-19T21:33:35.107 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-02-19T21:33:35.113 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-19T21:33:35.137 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi131.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-19 21:28:36.237824', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAgG0jlF18ihbqFbZzwpagx6+jBYHXzsEII8carzTu7Vw14Qs9Zo+UhC/Py6tfNnPNg7/sVImU3iV+Oq5Wzpj5I='} 2024-02-19T21:33:35.160 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi196.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-19 21:28:36.236132', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPFvA3EACYn4fifLnkd2R05kSwjMJ7iZwClMGYRZBS3L2isJojg/yFH9RzeBx+ZqIrNJlHzLANMvIdsslznGB9s='} 2024-02-19T21:33:35.161 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-02-19T21:33:35.166 INFO:teuthology.task.internal:roles: ubuntu@smithi131.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-02-19T21:33:35.166 INFO:teuthology.task.internal:roles: ubuntu@smithi196.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-02-19T21:33:35.166 INFO:teuthology.run_tasks:Running task console_log... 2024-02-19T21:33:35.236 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fef8ed35ee0>, signals=[15]) 2024-02-19T21:33:35.236 INFO:teuthology.run_tasks:Running task internal.connect... 2024-02-19T21:33:35.243 INFO:teuthology.task.internal:Opening connections... 2024-02-19T21:33:35.243 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:33:35.245 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi131.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:33:35.324 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi196.front.sepia.ceph.com 2024-02-19T21:33:35.325 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:33:35.393 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-02-19T21:33:35.400 DEBUG:teuthology.orchestra.run.smithi131:> uname -m 2024-02-19T21:33:35.422 INFO:teuthology.orchestra.run.smithi131.stdout:x86_64 2024-02-19T21:33:35.422 DEBUG:teuthology.orchestra.run.smithi131:> cat /etc/os-release 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:NAME="CentOS Stream" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:VERSION="8" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:ID="centos" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:ID_LIKE="rhel fedora" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:VERSION_ID="8" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:PLATFORM_ID="platform:el8" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:PRETTY_NAME="CentOS Stream 8" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:ANSI_COLOR="0;31" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-02-19T21:33:35.484 INFO:teuthology.orchestra.run.smithi131.stdout:HOME_URL="https://centos.org/" 2024-02-19T21:33:35.485 INFO:teuthology.orchestra.run.smithi131.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-02-19T21:33:35.485 INFO:teuthology.orchestra.run.smithi131.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-02-19T21:33:35.485 INFO:teuthology.orchestra.run.smithi131.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-02-19T21:33:35.485 INFO:teuthology.lock.ops:Updating smithi131.front.sepia.ceph.com on lock server 2024-02-19T21:33:35.505 DEBUG:teuthology.orchestra.run.smithi196:> uname -m 2024-02-19T21:33:35.526 INFO:teuthology.orchestra.run.smithi196.stdout:x86_64 2024-02-19T21:33:35.527 DEBUG:teuthology.orchestra.run.smithi196:> cat /etc/os-release 2024-02-19T21:33:35.587 INFO:teuthology.orchestra.run.smithi196.stdout:NAME="CentOS Stream" 2024-02-19T21:33:35.587 INFO:teuthology.orchestra.run.smithi196.stdout:VERSION="8" 2024-02-19T21:33:35.587 INFO:teuthology.orchestra.run.smithi196.stdout:ID="centos" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:ID_LIKE="rhel fedora" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:VERSION_ID="8" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:PLATFORM_ID="platform:el8" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:PRETTY_NAME="CentOS Stream 8" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:ANSI_COLOR="0;31" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:HOME_URL="https://centos.org/" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-02-19T21:33:35.588 INFO:teuthology.orchestra.run.smithi196.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-02-19T21:33:35.588 INFO:teuthology.lock.ops:Updating smithi196.front.sepia.ceph.com on lock server 2024-02-19T21:33:35.613 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-02-19T21:33:35.621 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-02-19T21:33:35.628 INFO:teuthology.task.internal:Checking for old test directory... 2024-02-19T21:33:35.628 DEBUG:teuthology.orchestra.run.smithi131:> test '!' -e /home/ubuntu/cephtest 2024-02-19T21:33:35.630 DEBUG:teuthology.orchestra.run.smithi196:> test '!' -e /home/ubuntu/cephtest 2024-02-19T21:33:35.650 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-02-19T21:33:35.656 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-02-19T21:33:35.656 DEBUG:teuthology.orchestra.run.smithi131:> test -z $(ls -A /var/lib/ceph) 2024-02-19T21:33:35.692 DEBUG:teuthology.orchestra.run.smithi196:> test -z $(ls -A /var/lib/ceph) 2024-02-19T21:33:35.723 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-02-19T21:33:35.776 INFO:teuthology.run_tasks:Running task kernel... 2024-02-19T21:33:35.791 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-02-19T21:33:35.791 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-02-19T21:33:35.791 DEBUG:teuthology.orchestra.run.smithi131:> test -f /run/.containerenv -o -f /.dockerenv 2024-02-19T21:33:35.792 DEBUG:teuthology.orchestra.run.smithi196:> test -f /run/.containerenv -o -f /.dockerenv 2024-02-19T21:33:35.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-19T21:33:35.811 DEBUG:teuthology.orchestra.run.smithi196:> uname -r 2024-02-19T21:33:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-19T21:33:35.812 DEBUG:teuthology.orchestra.run.smithi131:> uname -r 2024-02-19T21:33:35.871 INFO:teuthology.orchestra.run.smithi196.stdout:4.18.0-489.el8.x86_64 2024-02-19T21:33:35.871 INFO:teuthology.task.kernel:Running kernel on smithi196: 4.18.0-489.el8.x86_64 2024-02-19T21:33:35.871 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum install -y kernel 2024-02-19T21:33:35.873 INFO:teuthology.orchestra.run.smithi131.stdout:4.18.0-489.el8.x86_64 2024-02-19T21:33:35.873 INFO:teuthology.task.kernel:Running kernel on smithi131: 4.18.0-489.el8.x86_64 2024-02-19T21:33:35.874 DEBUG:teuthology.orchestra.run.smithi131:> sudo yum install -y kernel 2024-02-19T21:33:43.261 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 8 - AppStream 5.7 MB/s | 27 MB 00:04 2024-02-19T21:33:48.703 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 8 - BaseOS 17 MB/s | 10 MB 00:00 2024-02-19T21:33:50.437 INFO:teuthology.orchestra.run.smithi131.stdout:CentOS Stream 8 - AppStream 2.3 MB/s | 27 MB 00:11 2024-02-19T21:33:50.569 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 8 - Extras 69 kB/s | 18 kB 00:00 2024-02-19T21:33:51.045 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 8 - Extras common packages 19 kB/s | 6.9 kB 00:00 2024-02-19T21:33:51.355 INFO:teuthology.orchestra.run.smithi196.stdout:Copr repo for python3-asyncssh owned by ceph 16 kB/s | 3.5 kB 00:00 2024-02-19T21:33:56.095 INFO:teuthology.orchestra.run.smithi131.stdout:CentOS Stream 8 - BaseOS 14 MB/s | 10 MB 00:00 2024-02-19T21:33:57.954 INFO:teuthology.orchestra.run.smithi131.stdout:CentOS Stream 8 - Extras 67 kB/s | 18 kB 00:00 2024-02-19T21:33:58.346 INFO:teuthology.orchestra.run.smithi131.stdout:CentOS Stream 8 - Extras common packages 24 kB/s | 6.9 kB 00:00 2024-02-19T21:33:58.606 INFO:teuthology.orchestra.run.smithi131.stdout:Copr repo for python3-asyncssh owned by ceph 21 kB/s | 3.5 kB 00:00 2024-02-19T21:33:59.899 INFO:teuthology.orchestra.run.smithi131.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2024-02-19T21:34:03.599 INFO:teuthology.orchestra.run.smithi131.stdout:lab-extras 562 kB/s | 24 kB 00:00 2024-02-19T21:34:06.562 INFO:teuthology.orchestra.run.smithi131.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-02-19T21:34:06.563 INFO:teuthology.orchestra.run.smithi131.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-02-19T21:34:06.688 INFO:teuthology.orchestra.run.smithi131.stdout:Dependencies resolved. 2024-02-19T21:34:06.688 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:34:06.688 INFO:teuthology.orchestra.run.smithi131.stdout: Package Architecture Version Repository Size 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout:Installing: 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout: kernel x86_64 4.18.0-539.el8 baseos 10 M 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout:Installing dependencies: 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout: kernel-core x86_64 4.18.0-539.el8 baseos 43 M 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout: kernel-modules x86_64 4.18.0-539.el8 baseos 36 M 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout:Transaction Summary 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout:Install 3 Packages 2024-02-19T21:34:06.689 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:34:06.690 INFO:teuthology.orchestra.run.smithi131.stdout:Total download size: 90 M 2024-02-19T21:34:06.690 INFO:teuthology.orchestra.run.smithi131.stdout:Installed size: 97 M 2024-02-19T21:34:06.690 INFO:teuthology.orchestra.run.smithi131.stdout:Downloading Packages: 2024-02-19T21:34:07.231 INFO:teuthology.orchestra.run.smithi196.stdout:Extra Packages for Enterprise Linux 1.0 MB/s | 16 MB 00:15 2024-02-19T21:34:07.617 INFO:teuthology.orchestra.run.smithi131.stdout:(1/3): kernel-4.18.0-539.el8.x86_64.rpm 14 MB/s | 10 MB 00:00 2024-02-19T21:34:08.767 INFO:teuthology.orchestra.run.smithi131.stdout:(2/3): kernel-modules-4.18.0-539.el8.x86_64.rpm 19 MB/s | 36 MB 00:01 2024-02-19T21:34:09.218 INFO:teuthology.orchestra.run.smithi131.stdout:(3/3): kernel-core-4.18.0-539.el8.x86_64.rpm 18 MB/s | 43 MB 00:02 2024-02-19T21:34:09.218 INFO:teuthology.orchestra.run.smithi131.stdout:-------------------------------------------------------------------------------- 2024-02-19T21:34:09.218 INFO:teuthology.orchestra.run.smithi131.stdout:Total 35 MB/s | 90 MB 00:02 2024-02-19T21:34:09.898 INFO:teuthology.orchestra.run.smithi131.stdout:Running transaction check 2024-02-19T21:34:10.317 INFO:teuthology.orchestra.run.smithi131.stdout:Transaction check succeeded. 2024-02-19T21:34:10.317 INFO:teuthology.orchestra.run.smithi131.stdout:Running transaction test 2024-02-19T21:34:10.995 INFO:teuthology.orchestra.run.smithi196.stdout:lab-extras 570 kB/s | 24 kB 00:00 2024-02-19T21:34:11.547 INFO:teuthology.orchestra.run.smithi131.stdout:Transaction test succeeded. 2024-02-19T21:34:11.568 INFO:teuthology.orchestra.run.smithi131.stdout:Running transaction 2024-02-19T21:34:13.874 INFO:teuthology.orchestra.run.smithi196.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-02-19T21:34:13.875 INFO:teuthology.orchestra.run.smithi196.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-02-19T21:34:14.017 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-02-19T21:34:14.018 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:34:14.018 INFO:teuthology.orchestra.run.smithi196.stdout: Package Architecture Version Repository Size 2024-02-19T21:34:14.018 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:34:14.018 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-02-19T21:34:14.018 INFO:teuthology.orchestra.run.smithi196.stdout: kernel x86_64 4.18.0-539.el8 baseos 10 M 2024-02-19T21:34:14.018 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-core x86_64 4.18.0-539.el8 baseos 43 M 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-modules x86_64 4.18.0-539.el8 baseos 36 M 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout:Install 3 Packages 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 90 M 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 97 M 2024-02-19T21:34:14.019 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-02-19T21:34:14.346 INFO:teuthology.orchestra.run.smithi131.stdout: Preparing : 1/1 2024-02-19T21:34:14.828 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : kernel-core-4.18.0-539.el8.x86_64 1/3 2024-02-19T21:34:15.362 INFO:teuthology.orchestra.run.smithi196.stdout:(1/3): kernel-4.18.0-539.el8.x86_64.rpm 8.3 MB/s | 10 MB 00:01 2024-02-19T21:34:17.037 INFO:teuthology.orchestra.run.smithi196.stdout:(2/3): kernel-core-4.18.0-539.el8.x86_64.rpm 15 MB/s | 43 MB 00:02 2024-02-19T21:34:18.304 INFO:teuthology.orchestra.run.smithi196.stdout:(3/3): kernel-modules-4.18.0-539.el8.x86_64.rpm 8.6 MB/s | 36 MB 00:04 2024-02-19T21:34:18.305 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-02-19T21:34:18.305 INFO:teuthology.orchestra.run.smithi196.stdout:Total 21 MB/s | 90 MB 00:04 2024-02-19T21:34:18.407 INFO:teuthology.orchestra.run.smithi131.stdout: Running scriptlet: kernel-core-4.18.0-539.el8.x86_64 1/3 2024-02-19T21:34:18.786 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : kernel-modules-4.18.0-539.el8.x86_64 2/3 2024-02-19T21:34:18.990 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-02-19T21:34:19.414 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-02-19T21:34:19.414 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-02-19T21:34:20.657 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-02-19T21:34:20.680 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-02-19T21:34:22.231 INFO:teuthology.orchestra.run.smithi131.stdout: Running scriptlet: kernel-modules-4.18.0-539.el8.x86_64 2/3 2024-02-19T21:34:22.552 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : kernel-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:23.490 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-02-19T21:34:23.966 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : kernel-core-4.18.0-539.el8.x86_64 1/3 2024-02-19T21:34:27.517 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-core-4.18.0-539.el8.x86_64 1/3 2024-02-19T21:34:27.891 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : kernel-modules-4.18.0-539.el8.x86_64 2/3 2024-02-19T21:34:31.325 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-modules-4.18.0-539.el8.x86_64 2/3 2024-02-19T21:34:31.622 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : kernel-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:44.510 INFO:teuthology.orchestra.run.smithi131.stdout: Running scriptlet: kernel-core-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:44.533 INFO:teuthology.orchestra.run.smithi131.stdout: Running scriptlet: kernel-modules-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:44.876 INFO:teuthology.orchestra.run.smithi131.stdout: Running scriptlet: kernel-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:44.876 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : kernel-4.18.0-539.el8.x86_64 1/3 2024-02-19T21:34:44.876 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : kernel-core-4.18.0-539.el8.x86_64 2/3 2024-02-19T21:34:45.330 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : kernel-modules-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:45.330 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:34:45.330 INFO:teuthology.orchestra.run.smithi131.stdout:Installed: 2024-02-19T21:34:45.330 INFO:teuthology.orchestra.run.smithi131.stdout: kernel-4.18.0-539.el8.x86_64 kernel-core-4.18.0-539.el8.x86_64 2024-02-19T21:34:45.330 INFO:teuthology.orchestra.run.smithi131.stdout: kernel-modules-4.18.0-539.el8.x86_64 2024-02-19T21:34:45.330 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:34:45.330 INFO:teuthology.orchestra.run.smithi131.stdout:Complete! 2024-02-19T21:34:45.465 DEBUG:teuthology.orchestra.run.smithi131:> rpm -q kernel | sort -rV | head -n 1 2024-02-19T21:34:45.683 INFO:teuthology.orchestra.run.smithi131.stdout:kernel-4.18.0-539.el8.x86_64 2024-02-19T21:34:45.683 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-539.el8.x86_64 2024-02-19T21:34:45.683 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-539.el8.x86_64 2024-02-19T21:34:45.684 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-02-19T21:34:45.684 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-02-19T21:34:45.684 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-02-19T21:34:45.684 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi131.front.sepia.ceph.com, path=None, version=distro) 2024-02-19T21:34:45.684 DEBUG:teuthology.orchestra.run.smithi131:> rpm -q kernel | sort -rV | head -n 1 2024-02-19T21:34:45.895 INFO:teuthology.orchestra.run.smithi131.stdout:kernel-4.18.0-539.el8.x86_64 2024-02-19T21:34:45.895 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-539.el8.x86_64 2024-02-19T21:34:45.895 DEBUG:teuthology.orchestra.run.smithi131:> sudo rpm -qi grub2-tools 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Name : grub2-tools 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Epoch : 1 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Version : 2.02 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Release : 129.el8 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Architecture: x86_64 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Group : System Environment/Base 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Size : 9265586 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:License : GPLv3+ 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-02-19T21:34:45.929 INFO:teuthology.orchestra.run.smithi131.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:Build Host : kbuilder.bsys.centos.org 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:Relocations : (not relocatable) 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:Packager : CentOS BuildSystem 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:Vendor : CentOS 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:URL : http://www.gnu.org/software/grub/ 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:Summary : Support tools for GRUB. 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:Description : 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:customizable bootloader with modular architecture. It supports a rich 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:variety of kernel formats, file systems, computer architectures and 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:hardware devices. 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:34:45.930 INFO:teuthology.orchestra.run.smithi131.stdout:This subpackage provides tools for support of all platforms. 2024-02-19T21:34:45.931 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-02-19T21:34:45.931 INFO:teuthology.task.kernel:Updating grub on smithi131 to boot 4.18.0-539.el8.x86_64 2024-02-19T21:34:45.931 DEBUG:teuthology.orchestra.run.smithi131:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-02-19T21:34:47.761 INFO:teuthology.orchestra.run.smithi131.stderr:Generating grub configuration file ... 2024-02-19T21:34:49.937 INFO:teuthology.orchestra.run.smithi131.stderr:done 2024-02-19T21:34:49.938 DEBUG:teuthology.orchestra.run.smithi131:> mktemp 2024-02-19T21:34:49.971 INFO:teuthology.orchestra.run.smithi131.stdout:/tmp/tmp.g9iQr5dywJ 2024-02-19T21:34:49.971 DEBUG:teuthology.orchestra.run.smithi131:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.g9iQr5dywJ 2024-02-19T21:34:50.044 DEBUG:teuthology.orchestra.run.smithi131:> sudo chmod 0666 /tmp/tmp.g9iQr5dywJ 2024-02-19T21:34:50.193 DEBUG:teuthology.orchestra.remote:smithi131:/tmp/tmp.g9iQr5dywJ is 6KB 2024-02-19T21:34:50.216 DEBUG:teuthology.orchestra.run.smithi131:> rm -fr /tmp/tmp.g9iQr5dywJ 2024-02-19T21:34:50.233 DEBUG:teuthology.orchestra.run.smithi131:> sudo /bin/ls /boot/loader/entries || true 2024-02-19T21:34:50.302 INFO:teuthology.orchestra.run.smithi131.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-02-19T21:34:50.302 INFO:teuthology.orchestra.run.smithi131.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-02-19T21:34:50.302 INFO:teuthology.orchestra.run.smithi131.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-02-19T21:34:50.302 INFO:teuthology.orchestra.run.smithi131.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-539.el8.x86_64.conf 2024-02-19T21:34:50.303 DEBUG:teuthology.orchestra.run.smithi131:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-539.el8.x86_64 2024-02-19T21:34:50.460 DEBUG:teuthology.orchestra.run.smithi131:> sudo shutdown -r now 2024-02-19T21:34:53.486 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-core-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:53.510 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-modules-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:54.153 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:54.153 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : kernel-4.18.0-539.el8.x86_64 1/3 2024-02-19T21:34:54.154 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : kernel-core-4.18.0-539.el8.x86_64 2/3 2024-02-19T21:34:54.626 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : kernel-modules-4.18.0-539.el8.x86_64 3/3 2024-02-19T21:34:54.626 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:34:54.626 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-02-19T21:34:54.626 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-4.18.0-539.el8.x86_64 kernel-core-4.18.0-539.el8.x86_64 2024-02-19T21:34:54.626 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-modules-4.18.0-539.el8.x86_64 2024-02-19T21:34:54.627 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:34:54.627 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-02-19T21:34:54.765 DEBUG:teuthology.orchestra.run.smithi196:> rpm -q kernel | sort -rV | head -n 1 2024-02-19T21:34:54.983 INFO:teuthology.orchestra.run.smithi196.stdout:kernel-4.18.0-539.el8.x86_64 2024-02-19T21:34:54.984 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-539.el8.x86_64 2024-02-19T21:34:54.984 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-539.el8.x86_64 2024-02-19T21:34:54.984 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-02-19T21:34:54.984 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-02-19T21:34:54.984 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-02-19T21:34:54.984 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi196.front.sepia.ceph.com, path=None, version=distro) 2024-02-19T21:34:54.984 DEBUG:teuthology.orchestra.run.smithi196:> rpm -q kernel | sort -rV | head -n 1 2024-02-19T21:34:55.196 INFO:teuthology.orchestra.run.smithi196.stdout:kernel-4.18.0-539.el8.x86_64 2024-02-19T21:34:55.196 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-539.el8.x86_64 2024-02-19T21:34:55.196 DEBUG:teuthology.orchestra.run.smithi196:> sudo rpm -qi grub2-tools 2024-02-19T21:34:55.229 INFO:teuthology.orchestra.run.smithi196.stdout:Name : grub2-tools 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Epoch : 1 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Version : 2.02 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Release : 129.el8 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Architecture: x86_64 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Group : System Environment/Base 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Size : 9265586 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:License : GPLv3+ 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Build Host : kbuilder.bsys.centos.org 2024-02-19T21:34:55.230 INFO:teuthology.orchestra.run.smithi196.stdout:Relocations : (not relocatable) 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:Packager : CentOS BuildSystem 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:Vendor : CentOS 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:URL : http://www.gnu.org/software/grub/ 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:Summary : Support tools for GRUB. 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:Description : 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:customizable bootloader with modular architecture. It supports a rich 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:variety of kernel formats, file systems, computer architectures and 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:hardware devices. 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:34:55.231 INFO:teuthology.orchestra.run.smithi196.stdout:This subpackage provides tools for support of all platforms. 2024-02-19T21:34:55.232 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-02-19T21:34:55.232 INFO:teuthology.task.kernel:Updating grub on smithi196 to boot 4.18.0-539.el8.x86_64 2024-02-19T21:34:55.232 DEBUG:teuthology.orchestra.run.smithi196:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-02-19T21:34:56.668 INFO:teuthology.orchestra.run.smithi196.stderr:Generating grub configuration file ... 2024-02-19T21:34:58.496 INFO:teuthology.orchestra.run.smithi196.stderr:done 2024-02-19T21:34:58.498 DEBUG:teuthology.orchestra.run.smithi196:> mktemp 2024-02-19T21:34:58.521 INFO:teuthology.orchestra.run.smithi196.stdout:/tmp/tmp.ILjtVaiVa0 2024-02-19T21:34:58.522 DEBUG:teuthology.orchestra.run.smithi196:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ILjtVaiVa0 2024-02-19T21:34:58.605 DEBUG:teuthology.orchestra.run.smithi196:> sudo chmod 0666 /tmp/tmp.ILjtVaiVa0 2024-02-19T21:34:58.740 DEBUG:teuthology.orchestra.remote:smithi196:/tmp/tmp.ILjtVaiVa0 is 6KB 2024-02-19T21:34:58.789 DEBUG:teuthology.orchestra.run.smithi196:> rm -fr /tmp/tmp.ILjtVaiVa0 2024-02-19T21:34:58.806 DEBUG:teuthology.orchestra.run.smithi196:> sudo /bin/ls /boot/loader/entries || true 2024-02-19T21:34:58.877 INFO:teuthology.orchestra.run.smithi196.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-02-19T21:34:58.877 INFO:teuthology.orchestra.run.smithi196.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-02-19T21:34:58.877 INFO:teuthology.orchestra.run.smithi196.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-02-19T21:34:58.877 INFO:teuthology.orchestra.run.smithi196.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-539.el8.x86_64.conf 2024-02-19T21:34:58.878 DEBUG:teuthology.orchestra.run.smithi196:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-539.el8.x86_64 2024-02-19T21:34:59.076 DEBUG:teuthology.orchestra.run.smithi196:> sudo shutdown -r now 2024-02-19T21:35:20.485 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-02-19T21:35:20.486 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi131.front.sepia.ceph.com' 2024-02-19T21:35:20.487 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi131.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:35:29.088 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-02-19T21:35:29.088 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-02-19T21:35:29.089 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:35:38.812 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.131 2024-02-19T21:35:47.516 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.196 2024-02-19T21:35:47.813 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi131.front.sepia.ceph.com' 2024-02-19T21:35:47.814 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi131.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:35:56.524 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-02-19T21:35:56.524 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:35:59.580 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.196 2024-02-19T21:36:11.588 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-02-19T21:36:11.588 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:36:30.012 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.196 2024-02-19T21:36:45.026 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-02-19T21:36:45.027 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:36:46.102 DEBUG:teuthology.orchestra.run.smithi196:> true 2024-02-19T21:36:46.634 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-02-19T21:36:46.634 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "4.18.0-539.el8.x86_64"... 2024-02-19T21:36:46.635 DEBUG:teuthology.orchestra.run.smithi196:> uname -r 2024-02-19T21:36:46.651 INFO:teuthology.orchestra.run.smithi196.stdout:4.18.0-539.el8.x86_64 2024-02-19T21:36:46.652 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-539.el8.x86_64 vs 4.18.0-539.el8.x86_64 2024-02-19T21:36:46.652 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-02-19T21:36:46.652 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-02-19T21:36:47.653 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-02-19T21:36:47.654 DEBUG:teuthology.orchestra.run.smithi196:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-02-19T21:36:47.724 INFO:teuthology.orchestra.run.smithi196.stdout:ttyS1 2024-02-19T21:36:47.741 DEBUG:teuthology.parallel:result is None 2024-02-19T21:36:47.816 DEBUG:teuthology.orchestra.remote:timed out 2024-02-19T21:36:59.817 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi131.front.sepia.ceph.com' 2024-02-19T21:36:59.818 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi131.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:37:00.111 DEBUG:teuthology.orchestra.run.smithi131:> true 2024-02-19T21:37:00.402 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi131.front.sepia.ceph.com' 2024-02-19T21:37:00.402 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "4.18.0-539.el8.x86_64"... 2024-02-19T21:37:00.403 DEBUG:teuthology.orchestra.run.smithi131:> uname -r 2024-02-19T21:37:00.422 INFO:teuthology.orchestra.run.smithi131.stdout:4.18.0-539.el8.x86_64 2024-02-19T21:37:00.422 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-539.el8.x86_64 vs 4.18.0-539.el8.x86_64 2024-02-19T21:37:00.422 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-02-19T21:37:00.422 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-02-19T21:37:01.423 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-02-19T21:37:01.423 DEBUG:teuthology.orchestra.run.smithi131:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-02-19T21:37:01.490 INFO:teuthology.orchestra.run.smithi131.stdout:ttyS1 2024-02-19T21:37:01.506 DEBUG:teuthology.parallel:result is None 2024-02-19T21:37:01.507 INFO:teuthology.run_tasks:Running task internal.base... 2024-02-19T21:37:01.514 INFO:teuthology.task.internal:Creating test directory... 2024-02-19T21:37:01.514 DEBUG:teuthology.orchestra.run.smithi131:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-02-19T21:37:01.517 DEBUG:teuthology.orchestra.run.smithi196:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-02-19T21:37:01.538 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-02-19T21:37:01.545 INFO:teuthology.run_tasks:Running task internal.archive... 2024-02-19T21:37:01.553 INFO:teuthology.task.internal:Creating archive directory... 2024-02-19T21:37:01.553 DEBUG:teuthology.orchestra.run.smithi131:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-02-19T21:37:01.577 DEBUG:teuthology.orchestra.run.smithi196:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-02-19T21:37:01.610 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-02-19T21:37:01.617 INFO:teuthology.task.internal:Enabling coredump saving... 2024-02-19T21:37:01.617 DEBUG:teuthology.orchestra.run.smithi131:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-02-19T21:37:01.648 DEBUG:teuthology.orchestra.run.smithi196:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-02-19T21:37:01.683 INFO:teuthology.orchestra.run.smithi131.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-19T21:37:01.685 INFO:teuthology.orchestra.run.smithi196.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-19T21:37:01.693 INFO:teuthology.orchestra.run.smithi131.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-19T21:37:01.697 INFO:teuthology.orchestra.run.smithi196.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-19T21:37:01.699 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-02-19T21:37:01.706 INFO:teuthology.task.internal:Configuring sudo... 2024-02-19T21:37:01.706 DEBUG:teuthology.orchestra.run.smithi131:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-02-19T21:37:01.736 DEBUG:teuthology.orchestra.run.smithi196:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-02-19T21:37:01.770 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-02-19T21:37:01.779 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-02-19T21:37:01.779 DEBUG:teuthology.orchestra.run.smithi131:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-02-19T21:37:01.805 DEBUG:teuthology.orchestra.run.smithi196:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-02-19T21:37:01.829 DEBUG:teuthology.orchestra.run.smithi131:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-19T21:37:01.910 DEBUG:teuthology.orchestra.run.smithi131:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-19T21:37:01.964 DEBUG:teuthology.orchestra.run.smithi131:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-19T21:37:02.051 DEBUG:teuthology.orchestra.run.smithi131:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-19T21:37:02.091 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:37:02.091 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-02-19T21:37:02.158 DEBUG:teuthology.orchestra.run.smithi196:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-19T21:37:02.205 DEBUG:teuthology.orchestra.run.smithi196:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-19T21:37:02.264 DEBUG:teuthology.orchestra.run.smithi196:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-19T21:37:02.352 DEBUG:teuthology.orchestra.run.smithi196:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-19T21:37:02.398 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-02-19T21:37:02.399 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-02-19T21:37:02.468 DEBUG:teuthology.orchestra.run.smithi131:> sudo service rsyslog restart 2024-02-19T21:37:02.470 DEBUG:teuthology.orchestra.run.smithi196:> sudo service rsyslog restart 2024-02-19T21:37:02.531 INFO:teuthology.orchestra.run.smithi131.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-19T21:37:02.597 INFO:teuthology.orchestra.run.smithi196.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-19T21:37:02.938 INFO:teuthology.run_tasks:Running task internal.timer... 2024-02-19T21:37:02.945 INFO:teuthology.task.internal:Starting timer... 2024-02-19T21:37:02.946 INFO:teuthology.run_tasks:Running task pcp... 2024-02-19T21:37:02.956 INFO:teuthology.run_tasks:Running task selinux... 2024-02-19T21:37:02.965 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-02-19T21:37:02.965 DEBUG:teuthology.orchestra.run.smithi131:> sudo service auditd rotate 2024-02-19T21:37:03.049 INFO:teuthology.orchestra.run.smithi131.stdout:Rotating logs: 2024-02-19T21:37:03.050 DEBUG:teuthology.orchestra.run.smithi196:> sudo service auditd rotate 2024-02-19T21:37:03.166 INFO:teuthology.orchestra.run.smithi196.stdout:Rotating logs: 2024-02-19T21:37:03.168 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-02-19T21:37:03.168 DEBUG:teuthology.orchestra.run.smithi131:> /usr/sbin/getenforce 2024-02-19T21:37:03.209 INFO:teuthology.orchestra.run.smithi131.stdout:Permissive 2024-02-19T21:37:03.209 DEBUG:teuthology.orchestra.run.smithi196:> /usr/sbin/getenforce 2024-02-19T21:37:03.235 INFO:teuthology.orchestra.run.smithi196.stdout:Permissive 2024-02-19T21:37:03.235 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi131.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi196.front.sepia.ceph.com': 'permissive'} 2024-02-19T21:37:03.235 DEBUG:teuthology.orchestra.run.smithi131:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-19T21:37:03.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-19T21:37:03.276 DEBUG:teuthology.orchestra.run.smithi196:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-19T21:37:03.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-19T21:37:03.304 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-02-19T21:37:03.305 DEBUG:teuthology.orchestra.run.smithi131:> sudo /usr/sbin/setenforce permissive 2024-02-19T21:37:03.347 DEBUG:teuthology.orchestra.run.smithi196:> sudo /usr/sbin/setenforce permissive 2024-02-19T21:37:03.375 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-02-19T21:37:03.385 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-02-19T21:37:03.391 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-02-19T21:37:03.883 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-02-19T21:37:03.899 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-02-19T21:37:03.900 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi131.front.sepia.ceph.com,smithi196.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-02-19T21:42:59.062 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi131.front.sepia.ceph.com'), Remote(name='ubuntu@smithi196.front.sepia.ceph.com')] 2024-02-19T21:42:59.064 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi131.front.sepia.ceph.com' 2024-02-19T21:42:59.065 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi131.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:42:59.142 DEBUG:teuthology.orchestra.run.smithi131:> true 2024-02-19T21:42:59.166 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi131.front.sepia.ceph.com' 2024-02-19T21:42:59.167 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-02-19T21:42:59.167 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-19T21:42:59.248 DEBUG:teuthology.orchestra.run.smithi196:> true 2024-02-19T21:42:59.323 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-02-19T21:42:59.324 INFO:teuthology.run_tasks:Running task clock... 2024-02-19T21:42:59.335 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-02-19T21:42:59.335 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-02-19T21:42:59.336 DEBUG:teuthology.orchestra.run.smithi131:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-19T21:42:59.338 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-02-19T21:42:59.338 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-19T21:42:59.367 INFO:teuthology.orchestra.run.smithi131.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-02-19T21:42:59.380 INFO:teuthology.orchestra.run.smithi131.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-02-19T21:42:59.397 INFO:teuthology.orchestra.run.smithi196.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-02-19T21:42:59.412 INFO:teuthology.orchestra.run.smithi196.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-02-19T21:42:59.437 INFO:teuthology.orchestra.run.smithi131.stderr:sudo: ntpd: command not found 2024-02-19T21:42:59.448 INFO:teuthology.orchestra.run.smithi131.stdout:506 Cannot talk to daemon 2024-02-19T21:42:59.461 INFO:teuthology.orchestra.run.smithi131.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-02-19T21:42:59.474 INFO:teuthology.orchestra.run.smithi196.stderr:sudo: ntpd: command not found 2024-02-19T21:42:59.474 INFO:teuthology.orchestra.run.smithi131.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-02-19T21:42:59.484 INFO:teuthology.orchestra.run.smithi196.stdout:506 Cannot talk to daemon 2024-02-19T21:42:59.500 INFO:teuthology.orchestra.run.smithi196.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-02-19T21:42:59.511 INFO:teuthology.orchestra.run.smithi196.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-02-19T21:42:59.537 INFO:teuthology.orchestra.run.smithi131.stderr:bash: ntpq: command not found 2024-02-19T21:42:59.541 INFO:teuthology.orchestra.run.smithi131.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-19T21:42:59.542 INFO:teuthology.orchestra.run.smithi131.stdout:=============================================================================== 2024-02-19T21:42:59.542 INFO:teuthology.orchestra.run.smithi131.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T21:42:59.542 INFO:teuthology.orchestra.run.smithi131.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T21:42:59.542 INFO:teuthology.orchestra.run.smithi131.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T21:42:59.542 INFO:teuthology.orchestra.run.smithi131.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T21:42:59.573 INFO:teuthology.orchestra.run.smithi196.stderr:bash: ntpq: command not found 2024-02-19T21:42:59.577 INFO:teuthology.orchestra.run.smithi196.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-19T21:42:59.577 INFO:teuthology.orchestra.run.smithi196.stdout:=============================================================================== 2024-02-19T21:42:59.577 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T21:42:59.577 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T21:42:59.577 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T21:42:59.577 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T21:42:59.578 INFO:teuthology.run_tasks:Running task pexec... 2024-02-19T21:42:59.587 INFO:teuthology.task.pexec:Executing custom commands... 2024-02-19T21:42:59.587 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:42:59.587 DEBUG:teuthology.orchestra.run.smithi131:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-02-19T21:42:59.588 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi196.front.sepia.ceph.com 2024-02-19T21:42:59.588 DEBUG:teuthology.orchestra.run.smithi196:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-02-19T21:43:00.375 INFO:teuthology.orchestra.run.smithi131.stdout:Last metadata expiration check: 0:01:33 ago on Mon 19 Feb 2024 09:41:27 PM UTC. 2024-02-19T21:43:00.397 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:33 ago on Mon 19 Feb 2024 09:41:27 PM UTC. 2024-02-19T21:43:02.438 INFO:teuthology.orchestra.run.smithi131.stdout:Dependencies resolved. 2024-02-19T21:43:02.439 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:43:02.439 INFO:teuthology.orchestra.run.smithi131.stdout: Package Architecture Version Repository Size 2024-02-19T21:43:02.439 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:43:02.439 INFO:teuthology.orchestra.run.smithi131.stdout:Resetting modules: 2024-02-19T21:43:02.439 INFO:teuthology.orchestra.run.smithi131.stdout: container-tools 2024-02-19T21:43:02.439 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:43:02.440 INFO:teuthology.orchestra.run.smithi131.stdout:Transaction Summary 2024-02-19T21:43:02.440 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:43:02.440 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:43:02.440 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-02-19T21:43:02.441 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:43:02.441 INFO:teuthology.orchestra.run.smithi196.stdout: Package Architecture Version Repository Size 2024-02-19T21:43:02.441 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:43:02.441 INFO:teuthology.orchestra.run.smithi196.stdout:Resetting modules: 2024-02-19T21:43:02.441 INFO:teuthology.orchestra.run.smithi196.stdout: container-tools 2024-02-19T21:43:02.442 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:43:02.442 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-02-19T21:43:02.442 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:43:02.442 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:43:02.700 INFO:teuthology.orchestra.run.smithi131.stdout:Complete! 2024-02-19T21:43:02.732 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-02-19T21:43:03.573 INFO:teuthology.orchestra.run.smithi131.stdout:Last metadata expiration check: 0:01:36 ago on Mon 19 Feb 2024 09:41:27 PM UTC. 2024-02-19T21:43:03.591 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:36 ago on Mon 19 Feb 2024 09:41:27 PM UTC. 2024-02-19T21:43:05.614 INFO:teuthology.orchestra.run.smithi131.stdout:Dependencies resolved. 2024-02-19T21:43:05.625 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-02-19T21:43:05.650 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:43:05.650 INFO:teuthology.orchestra.run.smithi131.stdout: Package Arch Version Repository Size 2024-02-19T21:43:05.650 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout:Installing group/module packages: 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: aardvark-dns x86_64 2:1.10.0-1.module_el8+871+552be958 appstream 1.0 M 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: buildah x86_64 2:1.33.5-1.module_el8+885+7da147f3 appstream 9.7 M 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: cockpit-podman noarch 83-1.module_el8+847+7863d4e6 appstream 688 k 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: crun x86_64 1.14-1.module_el8+871+552be958 appstream 257 k 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: netavark x86_64 2:1.10.2-1.module_el8+885+7da147f3 appstream 4.3 M 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: python3-podman noarch 4.9.0-1.module_el8+847+7863d4e6 appstream 180 k 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: skopeo x86_64 2:1.14.3-0.1.module_el8+885+7da147f3 appstream 8.8 M 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: toolbox x86_64 0.0.99.5-1.module_el8+847+7863d4e6 appstream 2.5 M 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: udica noarch 0.2.6-20.module_el8+804+f131391c appstream 53 k 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout:Installing dependencies: 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2024-02-19T21:43:05.651 INFO:teuthology.orchestra.run.smithi131.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout:Installing weak dependencies: 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout:Installing module profiles: 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout: container-tools/common 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout:Enabling module streams: 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout: container-tools rhel8 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout:Transaction Summary 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout:================================================================================ 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout:Install 13 Packages 2024-02-19T21:43:05.652 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:43:05.653 INFO:teuthology.orchestra.run.smithi131.stdout:Total download size: 28 M 2024-02-19T21:43:05.653 INFO:teuthology.orchestra.run.smithi131.stdout:Installed size: 92 M 2024-02-19T21:43:05.653 INFO:teuthology.orchestra.run.smithi131.stdout:Downloading Packages: 2024-02-19T21:43:05.660 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:43:05.660 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-02-19T21:43:05.660 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:43:05.660 INFO:teuthology.orchestra.run.smithi196.stdout:Installing group/module packages: 2024-02-19T21:43:05.660 INFO:teuthology.orchestra.run.smithi196.stdout: aardvark-dns x86_64 2:1.10.0-1.module_el8+871+552be958 appstream 1.0 M 2024-02-19T21:43:05.660 INFO:teuthology.orchestra.run.smithi196.stdout: buildah x86_64 2:1.33.5-1.module_el8+885+7da147f3 appstream 9.7 M 2024-02-19T21:43:05.660 INFO:teuthology.orchestra.run.smithi196.stdout: cockpit-podman noarch 83-1.module_el8+847+7863d4e6 appstream 688 k 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: crun x86_64 1.14-1.module_el8+871+552be958 appstream 257 k 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: netavark x86_64 2:1.10.2-1.module_el8+885+7da147f3 appstream 4.3 M 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: python3-podman noarch 4.9.0-1.module_el8+847+7863d4e6 appstream 180 k 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: skopeo x86_64 2:1.14.3-0.1.module_el8+885+7da147f3 appstream 8.8 M 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: toolbox x86_64 0.0.99.5-1.module_el8+847+7863d4e6 appstream 2.5 M 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: udica noarch 0.2.6-20.module_el8+804+f131391c appstream 53 k 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout:Installing weak dependencies: 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout:Installing module profiles: 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: container-tools/common 2024-02-19T21:43:05.661 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:43:05.662 INFO:teuthology.orchestra.run.smithi196.stdout:Enabling module streams: 2024-02-19T21:43:05.662 INFO:teuthology.orchestra.run.smithi196.stdout: container-tools rhel8 2024-02-19T21:43:05.662 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:43:05.662 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-02-19T21:43:05.662 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-02-19T21:43:05.662 INFO:teuthology.orchestra.run.smithi196.stdout:Install 13 Packages 2024-02-19T21:43:05.662 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:43:05.663 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 28 M 2024-02-19T21:43:05.663 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 92 M 2024-02-19T21:43:05.663 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-02-19T21:43:06.392 INFO:teuthology.orchestra.run.smithi131.stdout:(1/13): cockpit-podman-83-1.module_el8+847+7863 1.5 MB/s | 688 kB 00:00 2024-02-19T21:43:06.470 INFO:teuthology.orchestra.run.smithi196.stdout:(1/13): cockpit-podman-83-1.module_el8+847+7863 1.1 MB/s | 688 kB 00:00 2024-02-19T21:43:06.484 INFO:teuthology.orchestra.run.smithi131.stdout:(2/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 413 kB/s | 38 kB 00:00 2024-02-19T21:43:06.562 INFO:teuthology.orchestra.run.smithi196.stdout:(2/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 413 kB/s | 38 kB 00:00 2024-02-19T21:43:06.777 INFO:teuthology.orchestra.run.smithi131.stdout:(3/13): crun-1.14-1.module_el8+871+552be958.x86 879 kB/s | 257 kB 00:00 2024-02-19T21:43:06.812 INFO:teuthology.orchestra.run.smithi196.stdout:(3/13): crun-1.14-1.module_el8+871+552be958.x86 1.0 MB/s | 257 kB 00:00 2024-02-19T21:43:06.862 INFO:teuthology.orchestra.run.smithi196.stdout:(4/13): aardvark-dns-1.10.0-1.module_el8+871+55 1.0 MB/s | 1.0 MB 00:01 2024-02-19T21:43:06.894 INFO:teuthology.orchestra.run.smithi131.stdout:(4/13): aardvark-dns-1.10.0-1.module_el8+871+55 1.1 MB/s | 1.0 MB 00:00 2024-02-19T21:43:07.046 INFO:teuthology.orchestra.run.smithi196.stdout:(5/13): python3-podman-4.9.0-1.module_el8+847+7 984 kB/s | 180 kB 00:00 2024-02-19T21:43:07.069 INFO:teuthology.orchestra.run.smithi131.stdout:(5/13): python3-podman-4.9.0-1.module_el8+847+7 1.0 MB/s | 180 kB 00:00 2024-02-19T21:43:07.136 INFO:teuthology.orchestra.run.smithi131.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 371 kB/s | 25 kB 00:00 2024-02-19T21:43:07.183 INFO:teuthology.orchestra.run.smithi196.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 181 kB/s | 25 kB 00:00 2024-02-19T21:43:07.236 INFO:teuthology.orchestra.run.smithi131.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 939 kB/s | 94 kB 00:00 2024-02-19T21:43:07.314 INFO:teuthology.orchestra.run.smithi196.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 722 kB/s | 94 kB 00:00 2024-02-19T21:43:12.119 INFO:teuthology.orchestra.run.smithi131.stdout:(8/13): netavark-1.10.2-1.module_el8+885+7da147 827 kB/s | 4.3 MB 00:05 2024-02-19T21:43:13.078 INFO:teuthology.orchestra.run.smithi196.stdout:(8/13): netavark-1.10.2-1.module_el8+885+7da147 705 kB/s | 4.3 MB 00:06 2024-02-19T21:43:14.777 INFO:teuthology.orchestra.run.smithi131.stdout:(9/13): toolbox-0.0.99.5-1.module_el8+847+7863d 974 kB/s | 2.5 MB 00:02 2024-02-19T21:43:14.869 INFO:teuthology.orchestra.run.smithi131.stdout:(10/13): udica-0.2.6-20.module_el8+804+f131391c 575 kB/s | 53 kB 00:00 2024-02-19T21:43:14.936 INFO:teuthology.orchestra.run.smithi131.stdout:(11/13): yajl-2.1.0-12.el8.x86_64.rpm 615 kB/s | 41 kB 00:00 2024-02-19T21:43:16.129 INFO:teuthology.orchestra.run.smithi196.stdout:(9/13): buildah-1.33.5-1.module_el8+885+7da147f 963 kB/s | 9.7 MB 00:10 2024-02-19T21:43:16.161 INFO:teuthology.orchestra.run.smithi131.stdout:(12/13): buildah-1.33.5-1.module_el8+885+7da147 968 kB/s | 9.7 MB 00:10 2024-02-19T21:43:16.204 INFO:teuthology.orchestra.run.smithi196.stdout:(10/13): toolbox-0.0.99.5-1.module_el8+847+7863 828 kB/s | 2.5 MB 00:03 2024-02-19T21:43:16.238 INFO:teuthology.orchestra.run.smithi196.stdout:(11/13): udica-0.2.6-20.module_el8+804+f131391c 486 kB/s | 53 kB 00:00 2024-02-19T21:43:16.288 INFO:teuthology.orchestra.run.smithi196.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 491 kB/s | 41 kB 00:00 2024-02-19T21:43:17.379 INFO:teuthology.orchestra.run.smithi196.stdout:(13/13): skopeo-1.14.3-0.1.module_el8+885+7da14 896 kB/s | 8.8 MB 00:10 2024-02-19T21:43:17.380 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-02-19T21:43:17.380 INFO:teuthology.orchestra.run.smithi196.stdout:Total 2.4 MB/s | 28 MB 00:11 2024-02-19T21:43:17.487 INFO:teuthology.orchestra.run.smithi131.stdout:(13/13): skopeo-1.14.3-0.1.module_el8+885+7da14 880 kB/s | 8.8 MB 00:10 2024-02-19T21:43:17.487 INFO:teuthology.orchestra.run.smithi131.stdout:-------------------------------------------------------------------------------- 2024-02-19T21:43:17.487 INFO:teuthology.orchestra.run.smithi131.stdout:Total 2.3 MB/s | 28 MB 00:11 2024-02-19T21:43:17.651 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-02-19T21:43:17.672 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-02-19T21:43:17.672 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-02-19T21:43:17.762 INFO:teuthology.orchestra.run.smithi131.stdout:Running transaction check 2024-02-19T21:43:17.783 INFO:teuthology.orchestra.run.smithi131.stdout:Transaction check succeeded. 2024-02-19T21:43:17.783 INFO:teuthology.orchestra.run.smithi131.stdout:Running transaction test 2024-02-19T21:43:17.928 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-02-19T21:43:17.931 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-02-19T21:43:18.042 INFO:teuthology.orchestra.run.smithi131.stdout:Transaction test succeeded. 2024-02-19T21:43:18.046 INFO:teuthology.orchestra.run.smithi131.stdout:Running transaction 2024-02-19T21:43:18.677 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-02-19T21:43:18.724 INFO:teuthology.orchestra.run.smithi131.stdout: Preparing : 1/1 2024-02-19T21:43:19.657 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2024-02-19T21:43:19.666 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2024-02-19T21:43:19.841 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 2/13 2024-02-19T21:43:19.872 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 2/13 2024-02-19T21:43:20.028 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/13 2024-02-19T21:43:20.034 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/13 2024-02-19T21:43:20.237 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 4/13 2024-02-19T21:43:20.240 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 4/13 2024-02-19T21:43:20.438 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 5/13 2024-02-19T21:43:20.477 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 5/13 2024-02-19T21:43:21.016 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x8 6/13 2024-02-19T21:43:21.079 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x8 6/13 2024-02-19T21:43:21.984 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : netavark-2:1.10.2-1.module_el8+885+7da147f3.x86_64 7/13 2024-02-19T21:43:22.080 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : netavark-2:1.10.2-1.module_el8+885+7da147f3.x86_64 7/13 2024-02-19T21:43:22.198 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : buildah-2:1.33.5-1.module_el8+885+7da147f3.x86_64 8/13 2024-02-19T21:43:22.295 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : buildah-2:1.33.5-1.module_el8+885+7da147f3.x86_64 8/13 2024-02-19T21:43:22.419 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : crun-1.14-1.module_el8+871+552be958.x86_64 9/13 2024-02-19T21:43:22.524 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : crun-1.14-1.module_el8+871+552be958.x86_64 9/13 2024-02-19T21:43:22.887 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : python3-podman-4.9.0-1.module_el8+847+7863d4e6.noa 10/13 2024-02-19T21:43:22.993 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-podman-4.9.0-1.module_el8+847+7863d4e6.noa 10/13 2024-02-19T21:43:23.089 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : toolbox-0.0.99.5-1.module_el8+847+7863d4e6.x86_64 11/13 2024-02-19T21:43:23.303 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : toolbox-0.0.99.5-1.module_el8+847+7863d4e6.x86_64 11/13 2024-02-19T21:43:23.330 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : udica-0.2.6-20.module_el8+804+f131391c.noarch 12/13 2024-02-19T21:43:23.494 INFO:teuthology.orchestra.run.smithi131.stdout: Installing : cockpit-podman-83-1.module_el8+847+7863d4e6.noarch 13/13 2024-02-19T21:43:23.521 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : udica-0.2.6-20.module_el8+804+f131391c.noarch 12/13 2024-02-19T21:43:23.757 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : cockpit-podman-83-1.module_el8+847+7863d4e6.noarch 13/13 2024-02-19T21:43:23.922 INFO:teuthology.orchestra.run.smithi131.stdout: Running scriptlet: cockpit-podman-83-1.module_el8+847+7863d4e6.noarch 13/13 2024-02-19T21:43:23.922 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x8 1/13 2024-02-19T21:43:23.922 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : buildah-2:1.33.5-1.module_el8+885+7da147f3.x86_64 2/13 2024-02-19T21:43:23.922 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : cockpit-podman-83-1.module_el8+847+7863d4e6.noarch 3/13 2024-02-19T21:43:23.922 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2024-02-19T21:43:23.922 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : crun-1.14-1.module_el8+871+552be958.x86_64 5/13 2024-02-19T21:43:23.922 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : netavark-2:1.10.2-1.module_el8+885+7da147f3.x86_64 6/13 2024-02-19T21:43:23.922 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : python3-podman-4.9.0-1.module_el8+847+7863d4e6.noa 7/13 2024-02-19T21:43:23.923 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2024-02-19T21:43:23.923 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2024-02-19T21:43:23.923 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 10/13 2024-02-19T21:43:23.923 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : toolbox-0.0.99.5-1.module_el8+847+7863d4e6.x86_64 11/13 2024-02-19T21:43:23.923 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : udica-0.2.6-20.module_el8+804+f131391c.noarch 12/13 2024-02-19T21:43:24.177 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: cockpit-podman-83-1.module_el8+847+7863d4e6.noarch 13/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x8 1/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : buildah-2:1.33.5-1.module_el8+885+7da147f3.x86_64 2/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : cockpit-podman-83-1.module_el8+847+7863d4e6.noarch 3/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : crun-1.14-1.module_el8+871+552be958.x86_64 5/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : netavark-2:1.10.2-1.module_el8+885+7da147f3.x86_64 6/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-podman-4.9.0-1.module_el8+847+7863d4e6.noa 7/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 10/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : toolbox-0.0.99.5-1.module_el8+847+7863d4e6.x86_64 11/13 2024-02-19T21:43:24.178 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : udica-0.2.6-20.module_el8+804+f131391c.noarch 12/13 2024-02-19T21:43:24.442 INFO:teuthology.orchestra.run.smithi131.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2024-02-19T21:43:24.442 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:43:24.442 INFO:teuthology.orchestra.run.smithi131.stdout:Installed: 2024-02-19T21:43:24.442 INFO:teuthology.orchestra.run.smithi131.stdout: aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x86_64 2024-02-19T21:43:24.442 INFO:teuthology.orchestra.run.smithi131.stdout: buildah-2:1.33.5-1.module_el8+885+7da147f3.x86_64 2024-02-19T21:43:24.442 INFO:teuthology.orchestra.run.smithi131.stdout: cockpit-podman-83-1.module_el8+847+7863d4e6.noarch 2024-02-19T21:43:24.442 INFO:teuthology.orchestra.run.smithi131.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: crun-1.14-1.module_el8+871+552be958.x86_64 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: netavark-2:1.10.2-1.module_el8+885+7da147f3.x86_64 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: python3-podman-4.9.0-1.module_el8+847+7863d4e6.noarch 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: python3-pyxdg-0.25-16.el8.noarch 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: toolbox-0.0.99.5-1.module_el8+847+7863d4e6.x86_64 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: udica-0.2.6-20.module_el8+804+f131391c.noarch 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: yajl-2.1.0-12.el8.x86_64 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:43:24.443 INFO:teuthology.orchestra.run.smithi131.stdout:Complete! 2024-02-19T21:43:24.680 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2024-02-19T21:43:24.680 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:43:24.680 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-02-19T21:43:24.681 INFO:teuthology.orchestra.run.smithi196.stdout: aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x86_64 2024-02-19T21:43:24.681 INFO:teuthology.orchestra.run.smithi196.stdout: buildah-2:1.33.5-1.module_el8+885+7da147f3.x86_64 2024-02-19T21:43:24.681 INFO:teuthology.orchestra.run.smithi196.stdout: cockpit-podman-83-1.module_el8+847+7863d4e6.noarch 2024-02-19T21:43:24.681 INFO:teuthology.orchestra.run.smithi196.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2024-02-19T21:43:24.681 INFO:teuthology.orchestra.run.smithi196.stdout: crun-1.14-1.module_el8+871+552be958.x86_64 2024-02-19T21:43:24.681 INFO:teuthology.orchestra.run.smithi196.stdout: netavark-2:1.10.2-1.module_el8+885+7da147f3.x86_64 2024-02-19T21:43:24.682 INFO:teuthology.orchestra.run.smithi196.stdout: python3-podman-4.9.0-1.module_el8+847+7863d4e6.noarch 2024-02-19T21:43:24.682 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2024-02-19T21:43:24.682 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pyxdg-0.25-16.el8.noarch 2024-02-19T21:43:24.682 INFO:teuthology.orchestra.run.smithi196.stdout: skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 2024-02-19T21:43:24.682 INFO:teuthology.orchestra.run.smithi196.stdout: toolbox-0.0.99.5-1.module_el8+847+7863d4e6.x86_64 2024-02-19T21:43:24.682 INFO:teuthology.orchestra.run.smithi196.stdout: udica-0.2.6-20.module_el8+804+f131391c.noarch 2024-02-19T21:43:24.683 INFO:teuthology.orchestra.run.smithi196.stdout: yajl-2.1.0-12.el8.x86_64 2024-02-19T21:43:24.683 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:43:24.683 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-02-19T21:43:24.688 DEBUG:teuthology.parallel:result is None 2024-02-19T21:43:24.983 DEBUG:teuthology.parallel:result is None 2024-02-19T21:43:24.983 INFO:teuthology.run_tasks:Running task cephadm... 2024-02-19T21:43:25.067 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v16.2.5', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'docker.io/ceph/ceph:v16.2.5', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '81bd20d634209c7cb82c18be12b4b5a05643ebf1'} 2024-02-19T21:43:25.067 INFO:tasks.cephadm:Cluster image is docker.io/ceph/ceph:v16.2.5 2024-02-19T21:43:25.068 INFO:tasks.cephadm:Cluster fsid is e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:43:25.068 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-02-19T21:43:25.068 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-02-19T21:43:25.068 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi131': '172.21.15.131', 'mon.smithi196': '172.21.15.196'} 2024-02-19T21:43:25.068 INFO:tasks.cephadm:Normalizing hostnames... 2024-02-19T21:43:25.069 DEBUG:teuthology.orchestra.run.smithi131:> sudo hostname $(hostname -s) 2024-02-19T21:43:25.097 DEBUG:teuthology.orchestra.run.smithi196:> sudo hostname $(hostname -s) 2024-02-19T21:43:25.125 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v16.2.5)... 2024-02-19T21:43:25.126 DEBUG:teuthology.orchestra.run.smithi131:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.5/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-02-19T21:43:25.395 INFO:teuthology.orchestra.run.smithi131.stdout:-rw-rw-r--. 1 ubuntu ubuntu 298325 Feb 19 21:43 /home/ubuntu/cephtest/cephadm 2024-02-19T21:43:25.395 DEBUG:teuthology.orchestra.run.smithi196:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.5/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-02-19T21:43:25.554 INFO:teuthology.orchestra.run.smithi196.stdout:-rw-rw-r--. 1 ubuntu ubuntu 298325 Feb 19 21:43 /home/ubuntu/cephtest/cephadm 2024-02-19T21:43:25.554 DEBUG:teuthology.orchestra.run.smithi131:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-02-19T21:43:25.573 DEBUG:teuthology.orchestra.run.smithi196:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-02-19T21:43:25.600 DEBUG:teuthology.orchestra.run.smithi131:> sudo mkdir -p /etc/ceph 2024-02-19T21:43:25.640 DEBUG:teuthology.orchestra.run.smithi196:> sudo mkdir -p /etc/ceph 2024-02-19T21:43:25.666 DEBUG:teuthology.orchestra.run.smithi131:> sudo chmod 777 /etc/ceph 2024-02-19T21:43:25.706 DEBUG:teuthology.orchestra.run.smithi196:> sudo chmod 777 /etc/ceph 2024-02-19T21:43:25.732 INFO:tasks.cephadm:Writing seed config... 2024-02-19T21:43:25.733 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-02-19T21:43:25.733 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-02-19T21:43:25.733 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-02-19T21:43:25.733 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-02-19T21:43:25.733 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-02-19T21:43:25.733 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-02-19T21:43:25.733 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-02-19T21:43:25.733 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-02-19T21:43:25.734 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:43:25.734 DEBUG:teuthology.orchestra.run.smithi131:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-02-19T21:43:25.763 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = e8e4a71a-cf6f-11ee-95bb-87774f69a715 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-02-19T21:43:25.763 DEBUG:teuthology.orchestra.run.smithi131:mon.smithi131> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131.service 2024-02-19T21:43:25.806 INFO:tasks.cephadm:Bootstrapping... 2024-02-19T21:43:25.806 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 -v bootstrap --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.131 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-02-19T21:43:25.832 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:-- Logs begin at Mon 2024-02-19 21:36:09 UTC. -- 2024-02-19T21:43:26.066 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2024-02-19T21:43:27.634 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: 4.9.3-dev 2024-02-19T21:43:27.651 INFO:teuthology.orchestra.run.smithi131.stderr:Verifying podman|docker is present... 2024-02-19T21:43:27.651 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2024-02-19T21:43:27.674 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: 4.9.3-dev 2024-02-19T21:43:27.688 INFO:teuthology.orchestra.run.smithi131.stderr:Verifying lvm2 is present... 2024-02-19T21:43:27.689 INFO:teuthology.orchestra.run.smithi131.stderr:Verifying time synchronization is in place... 2024-02-19T21:43:27.689 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-enabled chrony.service 2024-02-19T21:43:27.695 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-02-19T21:43:27.696 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-active chrony.service 2024-02-19T21:43:27.701 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: inactive 2024-02-19T21:43:27.702 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-enabled chronyd.service 2024-02-19T21:43:27.708 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: enabled 2024-02-19T21:43:27.708 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-active chronyd.service 2024-02-19T21:43:27.714 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: active 2024-02-19T21:43:27.714 INFO:teuthology.orchestra.run.smithi131.stderr:Unit chronyd.service is enabled and running 2024-02-19T21:43:27.715 INFO:teuthology.orchestra.run.smithi131.stderr:Repeating the final host check... 2024-02-19T21:43:27.715 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2024-02-19T21:43:27.735 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: 4.9.3-dev 2024-02-19T21:43:27.745 INFO:teuthology.orchestra.run.smithi131.stderr:podman|docker (/bin/podman) is present 2024-02-19T21:43:27.745 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl is present 2024-02-19T21:43:27.745 INFO:teuthology.orchestra.run.smithi131.stderr:lvcreate is present 2024-02-19T21:43:27.745 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-enabled chrony.service 2024-02-19T21:43:27.751 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-02-19T21:43:27.751 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-active chrony.service 2024-02-19T21:43:27.757 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: inactive 2024-02-19T21:43:27.757 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-enabled chronyd.service 2024-02-19T21:43:27.762 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: enabled 2024-02-19T21:43:27.763 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-active chronyd.service 2024-02-19T21:43:27.768 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: active 2024-02-19T21:43:27.769 INFO:teuthology.orchestra.run.smithi131.stderr:Unit chronyd.service is enabled and running 2024-02-19T21:43:27.769 INFO:teuthology.orchestra.run.smithi131.stderr:Host looks OK 2024-02-19T21:43:27.769 INFO:teuthology.orchestra.run.smithi131.stderr:Cluster fsid: e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:43:27.769 INFO:teuthology.orchestra.run.smithi131.stderr:Acquiring lock 140256232235136 on /run/cephadm/e8e4a71a-cf6f-11ee-95bb-87774f69a715.lock 2024-02-19T21:43:27.769 INFO:teuthology.orchestra.run.smithi131.stderr:Lock 140256232235136 acquired on /run/cephadm/e8e4a71a-cf6f-11ee-95bb-87774f69a715.lock 2024-02-19T21:43:27.770 INFO:teuthology.orchestra.run.smithi131.stderr:Verifying IP 172.21.15.131 port 3300 ... 2024-02-19T21:43:27.770 INFO:teuthology.orchestra.run.smithi131.stderr:Verifying IP 172.21.15.131 port 6789 ... 2024-02-19T21:43:27.770 INFO:teuthology.orchestra.run.smithi131.stderr:Base mon IP is 172.21.15.131, final addrv is [v2:172.21.15.131:3300,v1:172.21.15.131:6789] 2024-02-19T21:43:27.770 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /sbin/ip route ls 2024-02-19T21:43:27.772 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.131 metric 100 2024-02-19T21:43:27.772 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.131 metric 100 2024-02-19T21:43:27.773 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /sbin/ip -6 route ls 2024-02-19T21:43:27.775 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-02-19T21:43:27.775 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-02-19T21:43:27.775 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1799sec hoplimit 64 pref medium 2024-02-19T21:43:27.776 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /sbin/ip -6 addr ls 2024-02-19T21:43:27.778 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-02-19T21:43:27.778 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-02-19T21:43:27.778 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-02-19T21:43:27.778 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-02-19T21:43:27.778 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:72f9/64 scope link 2024-02-19T21:43:27.778 INFO:teuthology.orchestra.run.smithi131.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-02-19T21:43:27.779 INFO:teuthology.orchestra.run.smithi131.stderr:Mon IP `172.21.15.131` is in CIDR network `172.21.0.0/20` 2024-02-19T21:43:27.779 INFO:teuthology.orchestra.run.smithi131.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-02-19T21:43:27.781 INFO:teuthology.orchestra.run.smithi131.stderr:Pulling container image docker.io/ceph/ceph:v16.2.5... 2024-02-19T21:43:27.781 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman pull docker.io/ceph/ceph:v16.2.5 2024-02-19T21:43:27.876 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: Trying to pull docker.io/ceph/ceph:v16.2.5... 2024-02-19T21:43:34.467 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: Getting image source signatures 2024-02-19T21:43:34.470 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: Copying blob sha256:77890c5669f6b6c6924b8fca572f4f533c69239091810f2d2495b0df611f7263 2024-02-19T21:43:34.470 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: Copying blob sha256:7a0437f04f83f084b7ed68ad9c4a4947e12fc4e1b006b38129bac89114ec3621 2024-02-19T21:43:49.459 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: Copying config sha256:6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355 2024-02-19T21:43:49.617 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: Writing manifest to image destination 2024-02-19T21:43:49.934 INFO:teuthology.orchestra.run.smithi131.stderr:/bin/podman: 6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355 2024-02-19T21:43:49.943 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 --version 2024-02-19T21:43:50.553 INFO:teuthology.orchestra.run.smithi131.stderr:ceph: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable) 2024-02-19T21:43:51.187 INFO:teuthology.orchestra.run.smithi131.stderr:Ceph version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable) 2024-02-19T21:43:51.187 INFO:teuthology.orchestra.run.smithi131.stderr:Extracting ceph user uid/gid from container image... 2024-02-19T21:43:51.187 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 -c %u %g /var/lib/ceph 2024-02-19T21:43:51.738 INFO:teuthology.orchestra.run.smithi131.stderr:stat: 167 167 2024-02-19T21:43:52.338 INFO:teuthology.orchestra.run.smithi131.stderr:Creating initial keys... 2024-02-19T21:43:52.338 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 --gen-print-key 2024-02-19T21:43:52.903 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-authtool: AQCYy9Nlq2KcMBAA2Jk2UNH8goUT7dKTGOa5pA== 2024-02-19T21:43:53.491 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 --gen-print-key 2024-02-19T21:43:54.009 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-authtool: AQCZy9Nl/m51NxAAExam5Ce2wlsnrbBcO7PMLg== 2024-02-19T21:43:54.653 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.5 --gen-print-key 2024-02-19T21:43:55.191 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-authtool: AQCby9NlJk4RBBAAyRPX3dh+Ym9AM/U837IKlA== 2024-02-19T21:43:55.729 INFO:teuthology.orchestra.run.smithi131.stderr:Creating initial monmap... 2024-02-19T21:43:55.729 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/monmaptool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /tmp/ceph-tmp3y8e_old:/tmp/monmap:z docker.io/ceph/ceph:v16.2.5 --create --clobber --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 --addv smithi131 [v2:172.21.15.131:3300,v1:172.21.15.131:6789] /tmp/monmap 2024-02-19T21:43:56.248 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-02-19T21:43:56.248 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:43:56.248 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-02-19T21:43:56.826 INFO:teuthology.orchestra.run.smithi131.stderr:monmaptool for smithi131 [v2:172.21.15.131:3300,v1:172.21.15.131:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-02-19T21:43:56.826 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/monmaptool: set fsid to e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:43:56.826 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-02-19T21:43:56.826 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:43:56.826 INFO:teuthology.orchestra.run.smithi131.stderr:Creating mon... 2024-02-19T21:43:56.828 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph-mon --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131:/var/lib/ceph/mon/ceph-smithi131:z -v /tmp/ceph-tmp3uuft38l:/tmp/keyring:z -v /tmp/ceph-tmp3y8e_old:/tmp/monmap:z docker.io/ceph/ceph:v16.2.5 --mkfs -i smithi131 --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -c /dev/null --monmap /tmp/monmap --keyring /tmp/keyring --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true 2024-02-19T21:43:57.415 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.282+0000 7f428db64700 0 set uid:gid to 167:167 (ceph:ceph) 2024-02-19T21:43:57.415 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.283+0000 7f428db64700 1 imported monmap: 2024-02-19T21:43:57.415 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: epoch 0 2024-02-19T21:43:57.415 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:43:57.415 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: last_changed 2024-02-19T21:43:56.182677+0000 2024-02-19T21:43:57.415 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: created 2024-02-19T21:43:56.182677+0000 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: election_strategy: 1 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] mon.smithi131 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.283+0000 7f428db64700 0 /usr/bin/ceph-mon: set fsid to e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: RocksDB version: 6.8.1 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Compile date Jul 8 2021 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: DB SUMMARY 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi131/store.db dir, Total Num: 0, files: 2024-02-19T21:43:57.416 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi131/store.db: 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.error_if_exists: 0 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.create_if_missing: 1 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.paranoid_checks: 1 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.env: 0x55744ba9d1c0 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.fs: Posix File System 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.info_log: 0x55744d9aac20 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_file_opening_threads: 16 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.statistics: (nil) 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.use_fsync: 0 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_log_file_size: 0 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.log_file_time_to_roll: 0 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.keep_log_file_num: 1000 2024-02-19T21:43:57.417 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.recycle_log_file_num: 0 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.allow_fallocate: 1 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.allow_mmap_reads: 0 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.allow_mmap_writes: 0 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.use_direct_reads: 0 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.create_missing_column_families: 0 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.db_log_dir: 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi131/store.db 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.table_cache_numshardbits: 6 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_subcompactions: 1 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_background_flushes: -1 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-02-19T21:43:57.418 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.advise_random_on_open: 1 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.db_write_buffer_size: 0 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.write_buffer_manager: 0x55744d9b3530 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.use_adaptive_mutex: 0 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.rate_limiter: (nil) 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.wal_recovery_mode: 2 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.enable_thread_tracking: 0 2024-02-19T21:43:57.419 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.enable_pipelined_write: 0 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.unordered_write: 0 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.row_cache: None 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.wal_filter: None 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.allow_ingest_behind: 0 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.preserve_deletes: 0 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.two_write_queues: 0 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.manual_wal_flush: 0 2024-02-19T21:43:57.420 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.atomic_flush: 0 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.persist_stats_to_disk: 0 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.log_readahead_size: 0 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_background_jobs: 2 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_background_compactions: -1 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.delayed_write_rate : 16777216 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_total_wal_size: 0 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.stats_dump_period_sec: 600 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.stats_persist_period_sec: 600 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-19T21:43:57.421 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.max_open_files: -1 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.bytes_per_sync: 0 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.compaction_readahead_size: 0 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Compression algorithms supported: 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: kZSTD supported: 0 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: kXpressCompression supported: 0 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: kLZ4HCCompression supported: 1 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: kLZ4Compression supported: 1 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: kBZip2Compression supported: 0 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: kZlibCompression supported: 1 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: kSnappyCompression supported: 1 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2024-02-19T21:43:57.422 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi131/store.db/MANIFEST-000001 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.merge_operator: 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.compaction_filter: None 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.compaction_filter_factory: None 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.284+0000 7f428db64700 4 rocksdb: Options.table_factory: BlockBasedTable 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55744d8c4160) 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2024-02-19T21:43:57.423 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: index_type: 0 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: index_shortening: 1 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: checksum: 1 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: block_cache: 0x55744d8fad10 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: block_cache_options: 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: capacity : 536870912 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2024-02-19T21:43:57.424 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: block_size: 4096 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: partition_filters: 0 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: verify_compression: 0 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2024-02-19T21:43:57.425 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: format_version: 2 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: block_align: 0 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.write_buffer_size: 33554432 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_write_buffer_number: 2 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compression: NoCompression 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.bottommost_compression: Disabled 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.prefix_extractor: nullptr 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.num_levels: 7 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-19T21:43:57.426 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compression_opts.window_bits: -14 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compression_opts.level: 32767 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compression_opts.strategy: 0 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compression_opts.enabled: false 2024-02-19T21:43:57.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.target_file_size_base: 67108864 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.target_file_size_multiplier: 1 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-19T21:43:57.428 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.arena_block_size: 4194304 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.disable_auto_compactions: 0 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-19T21:43:57.429 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.table_properties_collectors: 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.inplace_update_support: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.memtable_huge_page_size: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.bloom_locality: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.max_successive_merges: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.paranoid_file_checks: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.force_consistency_checks: 0 2024-02-19T21:43:57.430 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.report_bg_io_stats: 0 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.ttl: 2592000 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi131/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.285+0000 7f428db64700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.286+0000 7f428db64700 4 rocksdb: DB pointer 0x55744d9bf800 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.286+0000 7f42769ed700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.286+0000 7f42769ed700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-19T21:43:57.431 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:43:57.432 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:43:57.433 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.434 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:43:57.435 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: 2024-02-19T21:43:57.435 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.287+0000 7f428db64700 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2024-02-19T21:43:57.435 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.287+0000 7f428db64700 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2024-02-19T21:43:57.435 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph-mon: debug 2024-02-19T21:43:57.287+0000 7f428db64700 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-smithi131 for mon.smithi131 2024-02-19T21:43:57.920 INFO:teuthology.orchestra.run.smithi131.stderr:create mon.smithi131 on 2024-02-19T21:43:57.920 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: install -d -m0770 -o 167 -g 167 /var/run/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:43:57.925 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl disable ceph.target 2024-02-19T21:43:58.016 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl enable ceph.target 2024-02-19T21:43:58.023 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-02-19T21:43:58.103 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl start ceph.target 2024-02-19T21:43:58.111 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl enable ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715.target 2024-02-19T21:43:58.118 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715.target → /etc/systemd/system/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715.target. 2024-02-19T21:43:58.118 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715.target → /etc/systemd/system/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715.target. 2024-02-19T21:43:58.199 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl start ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715.target 2024-02-19T21:43:58.206 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl daemon-reload 2024-02-19T21:43:58.290 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131 2024-02-19T21:43:58.313 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl reset-failed ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131 2024-02-19T21:43:58.318 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Failed to reset failed state of unit ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131.service: Unit ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131.service not loaded. 2024-02-19T21:43:58.318 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl enable ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131 2024-02-19T21:43:58.325 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Created symlink /etc/systemd/system/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715.target.wants/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131.service → /etc/systemd/system/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@.service. 2024-02-19T21:43:58.406 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl start ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131 2024-02-19T21:43:59.049 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-enabled firewalld.service 2024-02-19T21:43:59.056 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: disabled 2024-02-19T21:43:59.057 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-active firewalld.service 2024-02-19T21:43:59.063 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: inactive 2024-02-19T21:43:59.064 INFO:teuthology.orchestra.run.smithi131.stderr:firewalld.service is not enabled 2024-02-19T21:43:59.064 INFO:teuthology.orchestra.run.smithi131.stderr:Not possible to enable service . firewalld.service is not available 2024-02-19T21:43:59.065 INFO:teuthology.orchestra.run.smithi131.stderr:Waiting for mon to start... 2024-02-19T21:43:59.065 INFO:teuthology.orchestra.run.smithi131.stderr:Waiting for mon... 2024-02-19T21:43:59.065 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131:/var/lib/ceph/mon/ceph-smithi131:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 status 2024-02-19T21:43:59.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:43:59 smithi131 bash[34002]: 4b90fcc7a4e7aa22805d9843cd8d56d8455fb96df1924966651cdfbb75121589 2024-02-19T21:43:59.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:43:59 smithi131 systemd[1]: Started Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:43:59.662 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: cluster: 2024-02-19T21:43:59.662 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: id: e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: services: 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi131 (age 0.679672s) 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mgr: no daemons active 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: data: 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-02-19T21:43:59.663 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: pgs: 2024-02-19T21:43:59.664 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: cluster 2024-02-19T21:43:58.980968+0000 mon.smithi131 (mon.0) 2 : cluster [INF] mon.smithi131 is new leader, mons smithi131 in quorum (ranks 0) 2024-02-19T21:44:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: cluster 2024-02-19T21:43:58.981134+0000 mon.smithi131 (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {smithi131=[v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0]} 2024-02-19T21:44:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: cluster 2024-02-19T21:43:58.990040+0000 mon.smithi131 (mon.0) 4 : cluster [DBG] fsmap 2024-02-19T21:44:00.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: cluster 2024-02-19T21:43:58.999259+0000 mon.smithi131 (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-02-19T21:44:00.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: cluster 2024-02-19T21:43:58.999727+0000 mon.smithi131 (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2024-02-19T21:44:00.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: audit 2024-02-19T21:43:59.660609+0000 mon.smithi131 (mon.0) 7 : audit [DBG] from='client.? 172.21.15.131:0/3888733317' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-19T21:44:00.304 INFO:teuthology.orchestra.run.smithi131.stderr:mon is available 2024-02-19T21:44:00.305 INFO:teuthology.orchestra.run.smithi131.stderr:Assimilating anything we can from ceph.conf... 2024-02-19T21:44:00.305 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z -v /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131:/var/lib/ceph/mon/ceph-smithi131:z docker.io/ceph/ceph:v16.2.5 config assimilate-conf -i /var/lib/ceph/mon/ceph-smithi131/config 2024-02-19T21:44:00.879 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:00.879 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: [global] 2024-02-19T21:44:00.879 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: fsid = e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:44:00.879 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: log_to_journald = false 2024-02-19T21:44:00.879 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.131:3300,v1:172.21.15.131:6789] 2024-02-19T21:44:00.879 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-02-19T21:44:00.879 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-02-19T21:44:00.879 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-02-19T21:44:00.880 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-02-19T21:44:00.880 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:00.880 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: [mgr] 2024-02-19T21:44:00.880 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-02-19T21:44:00.880 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:00.880 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: [osd] 2024-02-19T21:44:00.880 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-02-19T21:44:00.880 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-02-19T21:44:01.216 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: audit 2024-02-19T21:44:00.875082+0000 mon.smithi131 (mon.0) 8 : audit [INF] from='client.? 172.21.15.131:0/3018617956' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-02-19T21:44:01.216 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: audit 2024-02-19T21:44:00.877675+0000 mon.smithi131 (mon.0) 9 : audit [INF] from='client.? 172.21.15.131:0/3018617956' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-02-19T21:44:01.418 INFO:teuthology.orchestra.run.smithi131.stderr:Generating new minimal ceph.conf... 2024-02-19T21:44:01.418 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z -v /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131:/var/lib/ceph/mon/ceph-smithi131:z docker.io/ceph/ceph:v16.2.5 config generate-minimal-conf -o /var/lib/ceph/mon/ceph-smithi131/config 2024-02-19T21:44:02.297 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: audit 2024-02-19T21:44:02.004581+0000 mon.smithi131 (mon.0) 10 : audit [DBG] from='client.? 172.21.15.131:0/2114067920' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:44:02.551 INFO:teuthology.orchestra.run.smithi131.stderr:Restarting the monitor... 2024-02-19T21:44:02.551 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl restart ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131 2024-02-19T21:44:02.976 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:02 smithi131 systemd[1]: Stopping Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:44:02.976 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: debug 2024-02-19T21:44:02.657+0000 7f8f78158700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi131 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-02-19T21:44:02.976 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34017]: debug 2024-02-19T21:44:02.657+0000 7f8f78158700 -1 mon.smithi131@0(leader) e1 *** Got Signal Terminated *** 2024-02-19T21:44:03.244 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 podman[34345]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131 2024-02-19T21:44:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131.service: Succeeded. 2024-02-19T21:44:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 systemd[1]: Stopped Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:44:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 systemd[1]: Starting Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:44:03.732 INFO:teuthology.orchestra.run.smithi131.stderr:Setting mon public_network to 172.21.0.0/20 2024-02-19T21:44:03.733 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 config set mon public_network 172.21.0.0/20 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.674+0000 7f82b6be0700 0 set uid:gid to 167:167 (ceph:ceph) 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.674+0000 7f82b6be0700 0 ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable), process ceph-mon, pid 7 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.674+0000 7f82b6be0700 0 pidfile_write: ignore empty --pid-file 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.678+0000 7f82b6be0700 0 load: jerasure load: lrc load: isa 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: RocksDB version: 6.8.1 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Compile date Jul 8 2021 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: DB SUMMARY 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: CURRENT file: CURRENT 2024-02-19T21:44:03.936 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: IDENTITY file: IDENTITY 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi131/store.db dir, Total Num: 1, files: 000004.sst 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi131/store.db: 000006.log size: 86225 ; 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.error_if_exists: 0 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.create_if_missing: 0 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.paranoid_checks: 1 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.env: 0x5599537a81c0 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.fs: Posix File System 2024-02-19T21:44:03.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.info_log: 0x559955fb1a40 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_file_opening_threads: 16 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.statistics: (nil) 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.use_fsync: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_log_file_size: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.log_file_time_to_roll: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.keep_log_file_num: 1000 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.recycle_log_file_num: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.allow_fallocate: 1 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.allow_mmap_reads: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.allow_mmap_writes: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.use_direct_reads: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.create_missing_column_families: 0 2024-02-19T21:44:03.938 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.db_log_dir: 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi131/store.db 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.table_cache_numshardbits: 6 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_subcompactions: 1 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_background_flushes: -1 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.advise_random_on_open: 1 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.db_write_buffer_size: 0 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.write_buffer_manager: 0x559955fb9fb0 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-19T21:44:03.939 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.use_adaptive_mutex: 0 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.rate_limiter: (nil) 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.wal_recovery_mode: 2 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.enable_thread_tracking: 0 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.enable_pipelined_write: 0 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.unordered_write: 0 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.row_cache: None 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.wal_filter: None 2024-02-19T21:44:03.940 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.allow_ingest_behind: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.preserve_deletes: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.two_write_queues: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.manual_wal_flush: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.atomic_flush: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.persist_stats_to_disk: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.log_readahead_size: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_background_jobs: 2 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_background_compactions: -1 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-19T21:44:03.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.delayed_write_rate : 16777216 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_total_wal_size: 0 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.stats_dump_period_sec: 600 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.stats_persist_period_sec: 600 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_open_files: -1 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bytes_per_sync: 0 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_readahead_size: 0 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Compression algorithms supported: 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: kZSTD supported: 0 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: kXpressCompression supported: 0 2024-02-19T21:44:03.942 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: kLZ4HCCompression supported: 1 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: kLZ4Compression supported: 1 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: kBZip2Compression supported: 0 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: kZlibCompression supported: 1 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: kSnappyCompression supported: 1 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi131/store.db/MANIFEST-000005 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.merge_operator: 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_filter: None 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_filter_factory: None 2024-02-19T21:44:03.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.table_factory: BlockBasedTable 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559955eca170) 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cache_index_and_filter_blocks: 1 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: pin_top_level_index_and_filter: 1 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: index_type: 0 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: data_block_index_type: 0 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: index_shortening: 1 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: data_block_hash_table_util_ratio: 0.750000 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: hash_index_allow_collision: 1 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: checksum: 1 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: no_block_cache: 0 2024-02-19T21:44:03.944 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: block_cache: 0x559955f00f10 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: block_cache_name: BinnedLRUCache 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: block_cache_options: 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: capacity : 536870912 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: num_shard_bits : 4 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: strict_capacity_limit : 0 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: high_pri_pool_ratio: 0.000 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: block_cache_compressed: (nil) 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: persistent_cache: (nil) 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: block_size: 4096 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: block_size_deviation: 10 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: block_restart_interval: 16 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: index_block_restart_interval: 1 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: metadata_block_size: 4096 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: partition_filters: 0 2024-02-19T21:44:03.945 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: use_delta_encoding: 1 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: whole_key_filtering: 1 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: verify_compression: 0 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: read_amp_bytes_per_bit: 0 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: format_version: 2 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: enable_index_compression: 1 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: block_align: 0 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.write_buffer_size: 33554432 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_write_buffer_number: 2 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compression: NoCompression 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bottommost_compression: Disabled 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.prefix_extractor: nullptr 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-19T21:44:03.946 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.num_levels: 7 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compression_opts.window_bits: -14 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compression_opts.level: 32767 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compression_opts.strategy: 0 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-19T21:44:03.947 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compression_opts.enabled: false 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.target_file_size_base: 67108864 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.target_file_size_multiplier: 1 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-19T21:44:03.948 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.arena_block_size: 4194304 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.disable_auto_compactions: 0 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-19T21:44:03.949 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-19T21:44:03.950 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.table_properties_collectors: 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.inplace_update_support: 0 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.memtable_huge_page_size: 0 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.bloom_locality: 0 2024-02-19T21:44:03.951 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.max_successive_merges: 0 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.paranoid_file_checks: 0 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.force_consistency_checks: 0 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.report_bg_io_stats: 0 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.ttl: 2592000 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.679+0000 7f82b6be0700 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.680+0000 7f82b6be0700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi131/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.680+0000 7f82b6be0700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.680+0000 7f82b6be0700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379043681197, "job": 1, "event": "recovery_started", "log_files": [6]} 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.680+0000 7f82b6be0700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2024-02-19T21:44:03.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.680+0000 7f82b6be0700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.681+0000 7f82b6be0700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379043681986, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 82956, "table_properties": {"data_size": 81214, "index_size": 347, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9462, "raw_average_key_size": 47, "raw_value_size": 75756, "raw_average_value_size": 378, "num_data_blocks": 11, "num_entries": 200, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1708379043, "oldest_key_time": 3, "file_creation_time": 0}} 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.681+0000 7f82b6be0700 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.681+0000 7f82b6be0700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379043682374, "job": 1, "event": "recovery_finished"} 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.682+0000 7f82b6be0700 4 rocksdb: DB pointer 0x559955fc5800 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.682+0000 7f829d9e8700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.682+0000 7f829d9e8700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ** DB Stats ** 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-19T21:44:03.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ** Compaction Stats [default] ** 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: L0 2/0 82.70 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.3 0.00 0.00 1 0.001 0 0 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Sum 2/0 82.70 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.3 0.00 0.00 1 0.001 0 0 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.3 0.00 0.00 1 0.001 0 0 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ** Compaction Stats [default] ** 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 149.3 0.00 0.00 1 0.001 0 0 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Flush(GB): cumulative 0.000, interval 0.000 2024-02-19T21:44:03.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Cumulative compaction: 0.00 GB write, 29.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Interval compaction: 0.00 GB write, 29.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ** Compaction Stats [default] ** 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:44:03.955 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: L0 2/0 82.70 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.3 0.00 0.00 1 0.001 0 0 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Sum 2/0 82.70 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.3 0.00 0.00 1 0.001 0 0 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ** Compaction Stats [default] ** 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 149.3 0.00 0.00 1 0.001 0 0 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Flush(GB): cumulative 0.000, interval 0.000 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Cumulative compaction: 0.00 GB write, 28.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:44:03.956 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.682+0000 7f82b6be0700 0 starting mon.smithi131 rank 0 at public addrs [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] at bind addrs [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi131 fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.683+0000 7f82b6be0700 1 mon.smithi131@-1(???) e1 preinit fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.683+0000 7f82b6be0700 0 mon.smithi131@-1(???).mds e1 new map 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.683+0000 7f82b6be0700 0 mon.smithi131@-1(???).mds e1 print_map 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: e1 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: enable_multiple, ever_enabled_multiple: 1,1 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: legacy client fscid: -1 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.957 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: No filesystems configured 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.684+0000 7f82b6be0700 0 mon.smithi131@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.684+0000 7f82b6be0700 0 mon.smithi131@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.684+0000 7f82b6be0700 0 mon.smithi131@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.684+0000 7f82b6be0700 0 mon.smithi131@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:02.004581+0000 mon.smithi131 (mon.0) 10 : audit [DBG] from='client.? 172.21.15.131:0/2114067920' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.684+0000 7f82b6be0700 1 mon.smithi131@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta expand map: {default=false} 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta from 'false' to 'false' 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta expanded map: {default=false} 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta expand map: {default=info} 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta from 'info' to 'info' 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta expanded map: {default=info} 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta expand map: {default=daemon} 2024-02-19T21:44:03.958 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta from 'daemon' to 'daemon' 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta expanded map: {default=daemon} 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta expand map: {default=debug} 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta from 'debug' to 'debug' 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:44:03.685+0000 7f82b6be0700 20 expand_channel_meta expanded map: {default=debug} 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 bash[34415]: d880a31c61701f603322a4452e2e729b3132a37a17c6746b8d63a328f627b339 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 systemd[1]: Started Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:03.700451+0000 mon.smithi131 (mon.0) 1 : cluster [INF] mon.smithi131 is new leader, mons smithi131 in quorum (ranks 0) 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:03.700510+0000 mon.smithi131 (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {smithi131=[v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0]} 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:03.700539+0000 mon.smithi131 (mon.0) 3 : cluster [DBG] fsmap 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:03.700561+0000 mon.smithi131 (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-02-19T21:44:03.959 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:03.700923+0000 mon.smithi131 (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2024-02-19T21:44:04.921 INFO:teuthology.orchestra.run.smithi131.stderr:Wrote config to /etc/ceph/ceph.conf 2024-02-19T21:44:04.921 INFO:teuthology.orchestra.run.smithi131.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-02-19T21:44:04.921 INFO:teuthology.orchestra.run.smithi131.stderr:Creating mgr... 2024-02-19T21:44:04.922 INFO:teuthology.orchestra.run.smithi131.stderr:Verifying port 9283 ... 2024-02-19T21:44:04.924 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl daemon-reload 2024-02-19T21:44:05.016 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mgr.smithi131.cjubsp 2024-02-19T21:44:05.038 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl reset-failed ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mgr.smithi131.cjubsp 2024-02-19T21:44:05.044 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Failed to reset failed state of unit ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mgr.smithi131.cjubsp.service: Unit ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mgr.smithi131.cjubsp.service not loaded. 2024-02-19T21:44:05.044 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl enable ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mgr.smithi131.cjubsp 2024-02-19T21:44:05.052 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: Created symlink /etc/systemd/system/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715.target.wants/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mgr.smithi131.cjubsp.service → /etc/systemd/system/ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@.service. 2024-02-19T21:44:05.133 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl start ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mgr.smithi131.cjubsp 2024-02-19T21:44:05.714 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:04.334580+0000 mon.smithi131 (mon.0) 6 : audit [INF] from='client.? 172.21.15.131:0/1534575170' entity='client.admin' 2024-02-19T21:44:05.718 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-enabled firewalld.service 2024-02-19T21:44:05.728 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: disabled 2024-02-19T21:44:05.729 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-active firewalld.service 2024-02-19T21:44:05.737 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: inactive 2024-02-19T21:44:05.738 INFO:teuthology.orchestra.run.smithi131.stderr:firewalld.service is not enabled 2024-02-19T21:44:05.738 INFO:teuthology.orchestra.run.smithi131.stderr:Not possible to enable service . firewalld.service is not available 2024-02-19T21:44:05.738 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-enabled firewalld.service 2024-02-19T21:44:05.747 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: disabled 2024-02-19T21:44:05.747 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-active firewalld.service 2024-02-19T21:44:05.754 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: inactive 2024-02-19T21:44:05.756 INFO:teuthology.orchestra.run.smithi131.stderr:firewalld.service is not enabled 2024-02-19T21:44:05.756 INFO:teuthology.orchestra.run.smithi131.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-02-19T21:44:05.756 INFO:teuthology.orchestra.run.smithi131.stderr:Waiting for mgr to start... 2024-02-19T21:44:05.756 INFO:teuthology.orchestra.run.smithi131.stderr:Waiting for mgr... 2024-02-19T21:44:05.756 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 status -f json-pretty 2024-02-19T21:44:06.446 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:06.446 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: { 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "fsid": "e8e4a71a-cf6f-11ee-95bb-87774f69a715", 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "health": { 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "checks": {}, 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "mutes": [] 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum": [ 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 0 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "smithi131" 2024-02-19T21:44:06.447 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "monmap": { 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osdmap": { 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-19T21:44:06.448 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-19T21:44:06.452 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-19T21:44:06.452 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "pgmap": { 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:06.453 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "fsmap": { 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "available": false, 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "modules": [ 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "iostat", 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "nfs", 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "restful" 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "services": {} 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:06.454 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "servicemap": { 2024-02-19T21:44:06.455 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:06.455 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "modified": "2024-02-19T21:43:58.984678+0000", 2024-02-19T21:44:06.455 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "services": {} 2024-02-19T21:44:06.455 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:06.455 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-19T21:44:06.455 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: } 2024-02-19T21:44:06.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:06.443771+0000 mon.smithi131 (mon.0) 7 : audit [DBG] from='client.? 172.21.15.131:0/2938328450' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-19T21:44:07.098 INFO:teuthology.orchestra.run.smithi131.stderr:mgr not available, waiting (1/15)... 2024-02-19T21:44:09.099 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 status -f json-pretty 2024-02-19T21:44:11.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:10.698550+0000 mon.smithi131 (mon.0) 8 : cluster [INF] Activating manager daemon smithi131.cjubsp 2024-02-19T21:44:11.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:10.701227+0000 mon.smithi131 (mon.0) 9 : cluster [DBG] mgrmap e2: smithi131.cjubsp(active, starting, since 0.00278482s) 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.701941+0000 mon.smithi131 (mon.0) 10 : audit [DBG] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.702072+0000 mon.smithi131 (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.702175+0000 mon.smithi131 (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.702287+0000 mon.smithi131 (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.702636+0000 mon.smithi131 (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi131.cjubsp", "id": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:10.704259+0000 mon.smithi131 (mon.0) 15 : cluster [INF] Manager daemon smithi131.cjubsp is now available 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.714360+0000 mon.smithi131 (mon.0) 16 : audit [INF] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.717812+0000 mon.smithi131 (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.718978+0000 mon.smithi131 (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.726022+0000 mon.smithi131 (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:11.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:10.731171+0000 mon.smithi131 (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.131:0/3264894962' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:11.697 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:11.697 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: { 2024-02-19T21:44:11.697 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "fsid": "e8e4a71a-cf6f-11ee-95bb-87774f69a715", 2024-02-19T21:44:11.697 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "health": { 2024-02-19T21:44:11.697 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "checks": {}, 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "mutes": [] 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum": [ 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 0 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "smithi131" 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum_age": 7, 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "monmap": { 2024-02-19T21:44:11.698 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osdmap": { 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-19T21:44:11.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-19T21:44:11.703 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-19T21:44:11.703 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-19T21:44:11.703 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:11.703 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "pgmap": { 2024-02-19T21:44:11.703 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-19T21:44:11.703 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-19T21:44:11.703 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-19T21:44:11.703 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "fsmap": { 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:11.704 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "available": false, 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "modules": [ 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "iostat", 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "nfs", 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "restful" 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "services": {} 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "servicemap": { 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "modified": "2024-02-19T21:43:58.984678+0000", 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "services": {} 2024-02-19T21:44:11.705 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:11.706 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-19T21:44:11.706 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: } 2024-02-19T21:44:12.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:11.694900+0000 mon.smithi131 (mon.0) 21 : audit [DBG] from='client.? 172.21.15.131:0/1627856394' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-19T21:44:12.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:11.703967+0000 mon.smithi131 (mon.0) 22 : cluster [DBG] mgrmap e3: smithi131.cjubsp(active, since 1.00552s) 2024-02-19T21:44:12.316 INFO:teuthology.orchestra.run.smithi131.stderr:mgr not available, waiting (2/15)... 2024-02-19T21:44:14.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:12.757925+0000 mon.smithi131 (mon.0) 23 : cluster [DBG] mgrmap e4: smithi131.cjubsp(active, since 2s) 2024-02-19T21:44:14.319 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 status -f json-pretty 2024-02-19T21:44:15.063 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:15.063 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: { 2024-02-19T21:44:15.063 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "fsid": "e8e4a71a-cf6f-11ee-95bb-87774f69a715", 2024-02-19T21:44:15.063 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "health": { 2024-02-19T21:44:15.063 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-19T21:44:15.063 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "checks": {}, 2024-02-19T21:44:15.063 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "mutes": [] 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum": [ 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 0 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "smithi131" 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "monmap": { 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:15.064 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osdmap": { 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "pgmap": { 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-19T21:44:15.065 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "fsmap": { 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "available": true, 2024-02-19T21:44:15.066 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "modules": [ 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "iostat", 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "nfs", 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "restful" 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ], 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "services": {} 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "servicemap": { 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "modified": "2024-02-19T21:43:58.984678+0000", 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "services": {} 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: }, 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-19T21:44:15.067 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: } 2024-02-19T21:44:15.386 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:15.057746+0000 mon.smithi131 (mon.0) 24 : audit [DBG] from='client.? 172.21.15.131:0/2777484981' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-19T21:44:15.631 INFO:teuthology.orchestra.run.smithi131.stderr:mgr is available 2024-02-19T21:44:15.632 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z -v /tmp/tmp_n6ho2yx:/var/lib/ceph/user.conf:z docker.io/ceph/ceph:v16.2.5 config assimilate-conf -i /var/lib/ceph/user.conf 2024-02-19T21:44:16.314 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:16.314 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: [global] 2024-02-19T21:44:16.314 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: fsid = e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:44:16.314 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: log_to_journald = false 2024-02-19T21:44:16.314 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-02-19T21:44:16.314 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: [mgr] 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: [osd] 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-02-19T21:44:16.315 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-02-19T21:44:16.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:16.312821+0000 mon.smithi131 (mon.0) 25 : audit [INF] from='client.? 172.21.15.131:0/2353714828' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-02-19T21:44:19.463 INFO:teuthology.orchestra.run.smithi131.stderr:Enabling cephadm module... 2024-02-19T21:44:19.463 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr module enable cephadm 2024-02-19T21:44:22.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:22.442595+0000 mon.smithi131 (mon.0) 26 : audit [INF] from='client.? 172.21.15.131:0/220105454' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-02-19T21:44:23.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:22.495860+0000 mon.smithi131 (mon.0) 27 : audit [INF] from='client.? 172.21.15.131:0/220105454' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-02-19T21:44:23.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:22.495932+0000 mon.smithi131 (mon.0) 28 : cluster [DBG] mgrmap e5: smithi131.cjubsp(active, since 11s) 2024-02-19T21:44:26.641 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr stat 2024-02-19T21:44:27.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: { 2024-02-19T21:44:27.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 5, 2024-02-19T21:44:27.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "available": true, 2024-02-19T21:44:27.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "active_name": "smithi131.cjubsp", 2024-02-19T21:44:27.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_standby": 0 2024-02-19T21:44:27.427 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: } 2024-02-19T21:44:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:27.426740+0000 mon.smithi131 (mon.0) 29 : audit [DBG] from='client.? 172.21.15.131:0/392201757' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-02-19T21:44:28.012 INFO:teuthology.orchestra.run.smithi131.stderr:Waiting for the mgr to restart... 2024-02-19T21:44:28.013 INFO:teuthology.orchestra.run.smithi131.stderr:Waiting for mgr epoch 5... 2024-02-19T21:44:28.013 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 tell mgr mgr_status 2024-02-19T21:44:28.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:27.578074+0000 mon.smithi131 (mon.0) 30 : cluster [INF] Active manager daemon smithi131.cjubsp restarted 2024-02-19T21:44:28.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:27.578517+0000 mon.smithi131 (mon.0) 31 : cluster [INF] Activating manager daemon smithi131.cjubsp 2024-02-19T21:44:28.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:27.584913+0000 mon.smithi131 (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2024-02-19T21:44:29.494 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: { 2024-02-19T21:44:29.494 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-02-19T21:44:29.494 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "initialized": true 2024-02-19T21:44:29.494 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: } 2024-02-19T21:44:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:28.483094+0000 mon.smithi131 (mon.0) 33 : cluster [DBG] mgrmap e6: smithi131.cjubsp(active, starting, since 0.904655s) 2024-02-19T21:44:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.483831+0000 mon.smithi131 (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.484566+0000 mon.smithi131 (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi131.cjubsp", "id": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.485452+0000 mon.smithi131 (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.485691+0000 mon.smithi131 (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.485902+0000 mon.smithi131 (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:28.499393+0000 mon.smithi131 (mon.0) 39 : cluster [INF] Manager daemon smithi131.cjubsp is now available 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.517994+0000 mon.smithi131 (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.520805+0000 mon.smithi131 (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.523356+0000 mon.smithi131 (mon.0) 42 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.612518+0000 mon.smithi131 (mon.0) 43 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.613202+0000 mon.smithi131 (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.614140+0000 mon.smithi131 (mon.0) 45 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.615604+0000 mon.smithi131 (mon.0) 46 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.626088+0000 mon.smithi131 (mon.0) 47 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:44:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:28.627213+0000 mon.smithi131 (mon.0) 48 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:44:30.112 INFO:teuthology.orchestra.run.smithi131.stderr:mgr epoch 5 is available 2024-02-19T21:44:30.112 INFO:teuthology.orchestra.run.smithi131.stderr:Setting orchestrator backend to cephadm... 2024-02-19T21:44:30.112 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch set backend cephadm 2024-02-19T21:44:30.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:29.487108+0000 mgr.smithi131.cjubsp (mgr.14116) 1 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-02-19T21:44:30.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:29.487597+0000 mon.smithi131 (mon.0) 49 : cluster [DBG] mgrmap e7: smithi131.cjubsp(active, since 1.90916s) 2024-02-19T21:44:30.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:29.493835+0000 mgr.smithi131.cjubsp (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-02-19T21:44:31.408 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 cephadm set-user root 2024-02-19T21:44:31.667 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:30.489606+0000 mon.smithi131 (mon.0) 50 : cluster [DBG] mgrmap e8: smithi131.cjubsp(active, since 2s) 2024-02-19T21:44:31.667 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:30.772001+0000 mon.smithi131 (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:31.667 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:30.775170+0000 mon.smithi131 (mon.0) 52 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:44:32.049 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: value unchanged 2024-02-19T21:44:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:30.767575+0000 mgr.smithi131.cjubsp (mgr.14116) 3 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:32.790 INFO:teuthology.orchestra.run.smithi131.stderr:Generating ssh key... 2024-02-19T21:44:32.790 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 cephadm generate-key 2024-02-19T21:44:33.742 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:32.049213+0000 mgr.smithi131.cjubsp (mgr.14116) 4 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:34.371 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 cephadm get-pub-key 2024-02-19T21:44:34.633 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:33.501026+0000 mgr.smithi131.cjubsp (mgr.14116) 6 : cephadm [INF] Generating ssh key... 2024-02-19T21:44:34.633 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:33.745252+0000 mon.smithi131 (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:34.633 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:33.746869+0000 mon.smithi131 (mon.0) 54 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:35.046 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWVkejctHpewH3j8O7WyDtuF9i1mTI6bBdr8SZHeMdsq+ABldg/VfoJQh5WUQicRUGwSBeyYML+SvWburJKeN/ztzLElURlJcXuJwOOC7s+LN7qxbOSv1uPcCA8plCRqa8mccIj1nvKgs2ZwNeZ7E0/CPbeLLYfjsT9AeBxmPJ4R72R5R4xc2grITDc2g6wvkhksH0iM2kr6TGbtvlcBHWVfsuJ43nPB0B497SVraKnPyi3EgIwXPt80/EwGKprhJUziCmxgUMiHBgqWDZhzmO4ki1ESFauqeVrVACI5jIrK1iDOm7w3SRyAo0iqwlrLK7HE1J3KrX7V8eXV9belLi7TM7iDh6ECRk/de/1UBQyJgOlhdQfGXf5G4ua5FmPlkY15x12NnCOq0MyuaqbH+YBzQl4oM3DpA9P/4TRIqjCQMdjMZbdF8dH3O2ZKiCOCfTWVaEJb/uKuBS58BIw4Oq+1QdyDAA4whSKAjd2oim3JItbeeDpv2sty8yCOmeXC8= ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:44:35.628 INFO:teuthology.orchestra.run.smithi131.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-02-19T21:44:35.628 INFO:teuthology.orchestra.run.smithi131.stderr:Adding key to root@localhost authorized_keys... 2024-02-19T21:44:35.629 INFO:teuthology.orchestra.run.smithi131.stderr:Adding host smithi131... 2024-02-19T21:44:35.629 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch host add smithi131 172.21.15.131 2024-02-19T21:44:35.630 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:35.046219+0000 mgr.smithi131.cjubsp (mgr.14116) 7 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:36.317495+0000 mgr.smithi131.cjubsp (mgr.14116) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi131", "addr": "172.21.15.131", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:36.906 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Added host 'smithi131' with addr '172.21.15.131' 2024-02-19T21:44:37.567 INFO:teuthology.orchestra.run.smithi131.stderr:Deploying mon service with default placement... 2024-02-19T21:44:37.568 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply mon 2024-02-19T21:44:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:36.904446+0000 mon.smithi131 (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:36.904729+0000 mgr.smithi131.cjubsp (mgr.14116) 9 : cephadm [INF] Added host smithi131 2024-02-19T21:44:38.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:36.904936+0000 mon.smithi131 (mon.0) 56 : audit [DBG] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:44:38.283 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Scheduled mon update... 2024-02-19T21:44:38.816 INFO:teuthology.orchestra.run.smithi131.stderr:Deploying mgr service with default placement... 2024-02-19T21:44:38.816 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply mgr 2024-02-19T21:44:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:38.280144+0000 mgr.smithi131.cjubsp (mgr.14116) 10 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:38.280875+0000 mgr.smithi131.cjubsp (mgr.14116) 11 : cephadm [INF] Saving service mon spec with placement count:5 2024-02-19T21:44:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:38.283210+0000 mon.smithi131 (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:39.554 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-02-19T21:44:40.131 INFO:teuthology.orchestra.run.smithi131.stderr:Deploying crash service with default placement... 2024-02-19T21:44:40.131 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply crash 2024-02-19T21:44:40.822 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Scheduled crash update... 2024-02-19T21:44:40.848 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:39.551357+0000 mgr.smithi131.cjubsp (mgr.14116) 12 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:40.848 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:39.551924+0000 mgr.smithi131.cjubsp (mgr.14116) 13 : cephadm [INF] Saving service mgr spec with placement count:2 2024-02-19T21:44:40.848 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:39.554162+0000 mon.smithi131 (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:41.403 INFO:teuthology.orchestra.run.smithi131.stderr:Enabling mgr prometheus module... 2024-02-19T21:44:41.404 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr module enable prometheus 2024-02-19T21:44:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:40.819691+0000 mgr.smithi131.cjubsp (mgr.14116) 14 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:40.820295+0000 mgr.smithi131.cjubsp (mgr.14116) 15 : cephadm [INF] Saving service crash spec with placement * 2024-02-19T21:44:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:40.822606+0000 mon.smithi131 (mon.0) 59 : audit [INF] from='mgr.14116 172.21.15.131:0/4062888160' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:43.158 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:42.125934+0000 mon.smithi131 (mon.0) 60 : audit [INF] from='client.? 172.21.15.131:0/3412964849' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-02-19T21:44:43.410 INFO:teuthology.orchestra.run.smithi131.stderr:Deploying prometheus service with default placement... 2024-02-19T21:44:43.410 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply prometheus 2024-02-19T21:44:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:42.828378+0000 mon.smithi131 (mon.0) 61 : audit [INF] from='client.? 172.21.15.131:0/3412964849' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-02-19T21:44:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:42.828454+0000 mon.smithi131 (mon.0) 62 : cluster [DBG] mgrmap e9: smithi131.cjubsp(active, since 15s) 2024-02-19T21:44:48.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:47.985841+0000 mon.smithi131 (mon.0) 63 : cluster [INF] Active manager daemon smithi131.cjubsp restarted 2024-02-19T21:44:48.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:47.986303+0000 mon.smithi131 (mon.0) 64 : cluster [INF] Activating manager daemon smithi131.cjubsp 2024-02-19T21:44:48.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:47.987947+0000 mon.smithi131 (mon.0) 65 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2024-02-19T21:44:49.052 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-02-19T21:44:49.373 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:48.038378+0000 mon.smithi131 (mon.0) 66 : cluster [DBG] mgrmap e10: smithi131.cjubsp(active, starting, since 0.0521682s) 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.040860+0000 mon.smithi131 (mon.0) 67 : audit [DBG] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.041575+0000 mon.smithi131 (mon.0) 68 : audit [DBG] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi131.cjubsp", "id": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.042495+0000 mon.smithi131 (mon.0) 69 : audit [DBG] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.042678+0000 mon.smithi131 (mon.0) 70 : audit [DBG] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.042825+0000 mon.smithi131 (mon.0) 71 : audit [DBG] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:48.056049+0000 mon.smithi131 (mon.0) 72 : cluster [INF] Manager daemon smithi131.cjubsp is now available 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.151993+0000 mon.smithi131 (mon.0) 73 : audit [DBG] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.153509+0000 mon.smithi131 (mon.0) 74 : audit [DBG] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.175017+0000 mon.smithi131 (mon.0) 75 : audit [INF] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:44:49.374 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:48.178022+0000 mon.smithi131 (mon.0) 76 : audit [INF] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:44:49.659 INFO:teuthology.orchestra.run.smithi131.stderr:Deploying grafana service with default placement... 2024-02-19T21:44:49.659 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply grafana 2024-02-19T21:44:50.367 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-02-19T21:44:50.389 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:49.046525+0000 mon.smithi131 (mon.0) 77 : cluster [DBG] mgrmap e11: smithi131.cjubsp(active, since 1.06031s) 2024-02-19T21:44:50.389 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:49.047861+0000 mgr.smithi131.cjubsp (mgr.14144) 2 : cephadm [INF] Saving service prometheus spec with placement count:1 2024-02-19T21:44:50.389 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:49.050672+0000 mon.smithi131 (mon.0) 78 : audit [INF] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:50.998 INFO:teuthology.orchestra.run.smithi131.stderr:Deploying node-exporter service with default placement... 2024-02-19T21:44:50.998 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply node-exporter 2024-02-19T21:44:51.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:50.052947+0000 mon.smithi131 (mon.0) 79 : cluster [DBG] mgrmap e12: smithi131.cjubsp(active, since 2s) 2024-02-19T21:44:51.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:50.364248+0000 mgr.smithi131.cjubsp (mgr.14144) 3 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:51.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:50.364781+0000 mgr.smithi131.cjubsp (mgr.14144) 4 : cephadm [INF] Saving service grafana spec with placement count:1 2024-02-19T21:44:51.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:50.366968+0000 mon.smithi131 (mon.0) 80 : audit [INF] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:51.699 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-02-19T21:44:52.280 INFO:teuthology.orchestra.run.smithi131.stderr:Deploying alertmanager service with default placement... 2024-02-19T21:44:52.280 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 orch apply alertmanager 2024-02-19T21:44:53.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:51.697086+0000 mgr.smithi131.cjubsp (mgr.14144) 5 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:53.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:51.697745+0000 mgr.smithi131.cjubsp (mgr.14144) 6 : cephadm [INF] Saving service node-exporter spec with placement * 2024-02-19T21:44:53.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:51.699972+0000 mon.smithi131 (mon.0) 81 : audit [INF] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:53.591 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-02-19T21:44:54.221 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 config set mgr mgr/cephadm/container_init True --force 2024-02-19T21:44:54.926 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:53.588859+0000 mgr.smithi131.cjubsp (mgr.14144) 7 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:44:54.926 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:44:53.589400+0000 mgr.smithi131.cjubsp (mgr.14144) 8 : cephadm [INF] Saving service alertmanager spec with placement count:1 2024-02-19T21:44:54.926 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:53.591481+0000 mon.smithi131 (mon.0) 82 : audit [INF] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:55.722 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 config set mgr mgr/dashboard/ssl_server_port 8443 2024-02-19T21:44:56.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:54.899810+0000 mon.smithi131 (mon.0) 83 : audit [INF] from='client.? 172.21.15.131:0/3676307060' entity='client.admin' 2024-02-19T21:44:57.642 INFO:teuthology.orchestra.run.smithi131.stderr:Enabling the dashboard module... 2024-02-19T21:44:57.642 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr module enable dashboard 2024-02-19T21:44:57.644 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:56.509714+0000 mon.smithi131 (mon.0) 84 : audit [INF] from='client.? 172.21.15.131:0/3332365195' entity='client.admin' 2024-02-19T21:44:57.644 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:57.093115+0000 mon.smithi131 (mon.0) 85 : audit [INF] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:59.243 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:57.940197+0000 mon.smithi131 (mon.0) 86 : audit [INF] from='mgr.14144 172.21.15.131:0/1498793062' entity='mgr.smithi131.cjubsp' 2024-02-19T21:44:59.243 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:58.430305+0000 mon.smithi131 (mon.0) 87 : audit [INF] from='client.? 172.21.15.131:0/2972478194' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-02-19T21:44:59.537 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 mgr stat 2024-02-19T21:45:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:44:58.944520+0000 mon.smithi131 (mon.0) 88 : audit [INF] from='client.? 172.21.15.131:0/2972478194' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-02-19T21:45:00.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:44:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:44:58.944568+0000 mon.smithi131 (mon.0) 89 : cluster [DBG] mgrmap e13: smithi131.cjubsp(active, since 10s) 2024-02-19T21:45:00.322 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: { 2024-02-19T21:45:00.322 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "epoch": 13, 2024-02-19T21:45:00.322 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "available": true, 2024-02-19T21:45:00.322 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "active_name": "smithi131.cjubsp", 2024-02-19T21:45:00.322 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "num_standby": 0 2024-02-19T21:45:00.322 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: } 2024-02-19T21:45:00.943 INFO:teuthology.orchestra.run.smithi131.stderr:Waiting for the mgr to restart... 2024-02-19T21:45:00.943 INFO:teuthology.orchestra.run.smithi131.stderr:Waiting for mgr epoch 13... 2024-02-19T21:45:00.943 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 tell mgr mgr_status 2024-02-19T21:45:01.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:00.322078+0000 mon.smithi131 (mon.0) 90 : audit [DBG] from='client.? 172.21.15.131:0/23516857' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-02-19T21:45:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:04.116946+0000 mon.smithi131 (mon.0) 91 : cluster [INF] Active manager daemon smithi131.cjubsp restarted 2024-02-19T21:45:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:04.117373+0000 mon.smithi131 (mon.0) 92 : cluster [INF] Activating manager daemon smithi131.cjubsp 2024-02-19T21:45:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:04.124214+0000 mon.smithi131 (mon.0) 93 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-02-19T21:45:05.713 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: { 2024-02-19T21:45:05.713 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "mgrmap_epoch": 15, 2024-02-19T21:45:05.713 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: "initialized": true 2024-02-19T21:45:05.713 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: } 2024-02-19T21:45:06.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:04.701107+0000 mon.smithi131 (mon.0) 94 : cluster [DBG] mgrmap e14: smithi131.cjubsp(active, starting, since 0.583823s) 2024-02-19T21:45:06.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.701919+0000 mon.smithi131 (mon.0) 95 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.702637+0000 mon.smithi131 (mon.0) 96 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi131.cjubsp", "id": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.703316+0000 mon.smithi131 (mon.0) 97 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.703467+0000 mon.smithi131 (mon.0) 98 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.703608+0000 mon.smithi131 (mon.0) 99 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:04.717319+0000 mon.smithi131 (mon.0) 100 : cluster [INF] Manager daemon smithi131.cjubsp is now available 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.815446+0000 mon.smithi131 (mon.0) 101 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.816527+0000 mon.smithi131 (mon.0) 102 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.841663+0000 mon.smithi131 (mon.0) 103 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:45:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:04.844261+0000 mon.smithi131 (mon.0) 104 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:45:06.509 INFO:teuthology.orchestra.run.smithi131.stderr:mgr epoch 13 is available 2024-02-19T21:45:06.509 INFO:teuthology.orchestra.run.smithi131.stderr:Generating a dashboard self-signed certificate... 2024-02-19T21:45:06.509 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 dashboard create-self-signed-cert 2024-02-19T21:45:06.791 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:05.706338+0000 mgr.smithi131.cjubsp (mgr.14164) 1 : audit [DBG] from='client.14168 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-02-19T21:45:06.791 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:05.706616+0000 mon.smithi131 (mon.0) 105 : cluster [DBG] mgrmap e15: smithi131.cjubsp(active, since 1.58933s) 2024-02-19T21:45:06.791 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:05.712648+0000 mgr.smithi131.cjubsp (mgr.14164) 2 : audit [DBG] from='client.14168 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-02-19T21:45:06.791 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:06.122464+0000 mon.smithi131 (mon.0) 106 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:07.386 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: Self-signed certificate created 2024-02-19T21:45:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:07.123867+0000 mon.smithi131 (mon.0) 107 : cluster [DBG] mgrmap e16: smithi131.cjubsp(active, since 3s) 2024-02-19T21:45:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:07.341183+0000 mgr.smithi131.cjubsp (mgr.14164) 3 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:45:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:07.383712+0000 mon.smithi131 (mon.0) 108 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:07.386261+0000 mon.smithi131 (mon.0) 109 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:08.547 INFO:teuthology.orchestra.run.smithi131.stderr:Creating initial admin user... 2024-02-19T21:45:08.547 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmplvm3xwzu:/tmp/dashboard.pw:z docker.io/ceph/ceph:v16.2.5 dashboard ac-user-create admin -i /tmp/dashboard.pw administrator --force-password --pwd-update-required 2024-02-19T21:45:09.487 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$bl1rMYGuXtNAob6BHCJA8u23G/pOoMMG8ZHwZAMPr8YiTBd4UI94.", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1708379109, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-02-19T21:45:10.079 INFO:teuthology.orchestra.run.smithi131.stderr:Fetching dashboard port number... 2024-02-19T21:45:10.079 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.5 -e NODE_NAME=smithi131 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715:/var/log/ceph:z -v /tmp/ceph-tmpo02epk3k:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmp29q23b85:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.5 config get mgr mgr/dashboard/ssl_server_port 2024-02-19T21:45:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:09.273302+0000 mgr.smithi131.cjubsp (mgr.14164) 4 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:45:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:09.487993+0000 mon.smithi131 (mon.0) 110 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:10.857 INFO:teuthology.orchestra.run.smithi131.stderr:/usr/bin/ceph: 8443 2024-02-19T21:45:11.485 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-enabled firewalld.service 2024-02-19T21:45:11.494 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: disabled 2024-02-19T21:45:11.494 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: systemctl is-active firewalld.service 2024-02-19T21:45:11.501 INFO:teuthology.orchestra.run.smithi131.stderr:systemctl: inactive 2024-02-19T21:45:11.502 INFO:teuthology.orchestra.run.smithi131.stderr:firewalld.service is not enabled 2024-02-19T21:45:11.502 INFO:teuthology.orchestra.run.smithi131.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-02-19T21:45:11.505 INFO:teuthology.orchestra.run.smithi131.stderr:Ceph Dashboard is now available at: 2024-02-19T21:45:11.505 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:45:11.505 INFO:teuthology.orchestra.run.smithi131.stderr: URL: https://smithi131.front.sepia.ceph.com:8443/ 2024-02-19T21:45:11.505 INFO:teuthology.orchestra.run.smithi131.stderr: User: admin 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: Password: bc7a1wxgss 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr:You can access the Ceph CLI with: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr:Please consider enabling telemetry to help improve Ceph: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: ceph telemetry on 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr:For more information see: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: https://docs.ceph.com/docs/pacific/mgr/telemetry/ 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T21:45:11.506 INFO:teuthology.orchestra.run.smithi131.stderr:Bootstrap complete. 2024-02-19T21:45:11.507 INFO:teuthology.orchestra.run.smithi131.stderr:Releasing lock 140256232235136 on /run/cephadm/e8e4a71a-cf6f-11ee-95bb-87774f69a715.lock 2024-02-19T21:45:11.507 INFO:teuthology.orchestra.run.smithi131.stderr:Lock 140256232235136 released on /run/cephadm/e8e4a71a-cf6f-11ee-95bb-87774f69a715.lock 2024-02-19T21:45:11.539 INFO:tasks.cephadm:Fetching config... 2024-02-19T21:45:11.539 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:45:11.539 DEBUG:teuthology.orchestra.run.smithi131:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-02-19T21:45:11.557 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-02-19T21:45:11.557 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:45:11.557 DEBUG:teuthology.orchestra.run.smithi131:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-02-19T21:45:11.615 INFO:tasks.cephadm:Fetching mon keyring... 2024-02-19T21:45:11.615 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:45:11.615 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd if=/var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/keyring of=/dev/stdout 2024-02-19T21:45:11.682 INFO:tasks.cephadm:Fetching pub ssh key... 2024-02-19T21:45:11.682 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:45:11.682 DEBUG:teuthology.orchestra.run.smithi131:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-02-19T21:45:11.740 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-02-19T21:45:11.741 DEBUG:teuthology.orchestra.run.smithi131:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWVkejctHpewH3j8O7WyDtuF9i1mTI6bBdr8SZHeMdsq+ABldg/VfoJQh5WUQicRUGwSBeyYML+SvWburJKeN/ztzLElURlJcXuJwOOC7s+LN7qxbOSv1uPcCA8plCRqa8mccIj1nvKgs2ZwNeZ7E0/CPbeLLYfjsT9AeBxmPJ4R72R5R4xc2grITDc2g6wvkhksH0iM2kr6TGbtvlcBHWVfsuJ43nPB0B497SVraKnPyi3EgIwXPt80/EwGKprhJUziCmxgUMiHBgqWDZhzmO4ki1ESFauqeVrVACI5jIrK1iDOm7w3SRyAo0iqwlrLK7HE1J3KrX7V8eXV9belLi7TM7iDh6ECRk/de/1UBQyJgOlhdQfGXf5G4ua5FmPlkY15x12NnCOq0MyuaqbH+YBzQl4oM3DpA9P/4TRIqjCQMdjMZbdF8dH3O2ZKiCOCfTWVaEJb/uKuBS58BIw4Oq+1QdyDAA4whSKAjd2oim3JItbeeDpv2sty8yCOmeXC8= ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-02-19T21:45:11.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:10.489253+0000 mon.smithi131 (mon.0) 111 : cluster [DBG] mgrmap e17: smithi131.cjubsp(active, since 6s) 2024-02-19T21:45:11.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:10.857376+0000 mon.smithi131 (mon.0) 112 : audit [DBG] from='client.? 172.21.15.131:0/3601388884' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-02-19T21:45:11.793 INFO:teuthology.orchestra.run.smithi131.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWVkejctHpewH3j8O7WyDtuF9i1mTI6bBdr8SZHeMdsq+ABldg/VfoJQh5WUQicRUGwSBeyYML+SvWburJKeN/ztzLElURlJcXuJwOOC7s+LN7qxbOSv1uPcCA8plCRqa8mccIj1nvKgs2ZwNeZ7E0/CPbeLLYfjsT9AeBxmPJ4R72R5R4xc2grITDc2g6wvkhksH0iM2kr6TGbtvlcBHWVfsuJ43nPB0B497SVraKnPyi3EgIwXPt80/EwGKprhJUziCmxgUMiHBgqWDZhzmO4ki1ESFauqeVrVACI5jIrK1iDOm7w3SRyAo0iqwlrLK7HE1J3KrX7V8eXV9belLi7TM7iDh6ECRk/de/1UBQyJgOlhdQfGXf5G4ua5FmPlkY15x12NnCOq0MyuaqbH+YBzQl4oM3DpA9P/4TRIqjCQMdjMZbdF8dH3O2ZKiCOCfTWVaEJb/uKuBS58BIw4Oq+1QdyDAA4whSKAjd2oim3JItbeeDpv2sty8yCOmeXC8= ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:45:11.804 DEBUG:teuthology.orchestra.run.smithi196:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWVkejctHpewH3j8O7WyDtuF9i1mTI6bBdr8SZHeMdsq+ABldg/VfoJQh5WUQicRUGwSBeyYML+SvWburJKeN/ztzLElURlJcXuJwOOC7s+LN7qxbOSv1uPcCA8plCRqa8mccIj1nvKgs2ZwNeZ7E0/CPbeLLYfjsT9AeBxmPJ4R72R5R4xc2grITDc2g6wvkhksH0iM2kr6TGbtvlcBHWVfsuJ43nPB0B497SVraKnPyi3EgIwXPt80/EwGKprhJUziCmxgUMiHBgqWDZhzmO4ki1ESFauqeVrVACI5jIrK1iDOm7w3SRyAo0iqwlrLK7HE1J3KrX7V8eXV9belLi7TM7iDh6ECRk/de/1UBQyJgOlhdQfGXf5G4ua5FmPlkY15x12NnCOq0MyuaqbH+YBzQl4oM3DpA9P/4TRIqjCQMdjMZbdF8dH3O2ZKiCOCfTWVaEJb/uKuBS58BIw4Oq+1QdyDAA4whSKAjd2oim3JItbeeDpv2sty8yCOmeXC8= ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-02-19T21:45:11.850 INFO:teuthology.orchestra.run.smithi196.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWVkejctHpewH3j8O7WyDtuF9i1mTI6bBdr8SZHeMdsq+ABldg/VfoJQh5WUQicRUGwSBeyYML+SvWburJKeN/ztzLElURlJcXuJwOOC7s+LN7qxbOSv1uPcCA8plCRqa8mccIj1nvKgs2ZwNeZ7E0/CPbeLLYfjsT9AeBxmPJ4R72R5R4xc2grITDc2g6wvkhksH0iM2kr6TGbtvlcBHWVfsuJ43nPB0B497SVraKnPyi3EgIwXPt80/EwGKprhJUziCmxgUMiHBgqWDZhzmO4ki1ESFauqeVrVACI5jIrK1iDOm7w3SRyAo0iqwlrLK7HE1J3KrX7V8eXV9belLi7TM7iDh6ECRk/de/1UBQyJgOlhdQfGXf5G4ua5FmPlkY15x12NnCOq0MyuaqbH+YBzQl4oM3DpA9P/4TRIqjCQMdjMZbdF8dH3O2ZKiCOCfTWVaEJb/uKuBS58BIw4Oq+1QdyDAA4whSKAjd2oim3JItbeeDpv2sty8yCOmeXC8= ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:45:11.862 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-02-19T21:45:15.151 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-02-19T21:45:15.151 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-02-19T21:45:15.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:14.149793+0000 mon.smithi131 (mon.0) 113 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:15.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:14.429824+0000 mon.smithi131 (mon.0) 114 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi131", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:45:15.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:14.433081+0000 mon.smithi131 (mon.0) 115 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:15.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:45:14.439651+0000 mgr.smithi131.cjubsp (mgr.14164) 5 : cephadm [INF] Deploying daemon alertmanager.smithi131 on smithi131 2024-02-19T21:45:15.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:14.527360+0000 mon.smithi131 (mon.0) 116 : audit [INF] from='client.? 172.21.15.131:0/1019628428' entity='client.admin' 2024-02-19T21:45:17.572 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi196 2024-02-19T21:45:17.573 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-02-19T21:45:17.573 DEBUG:teuthology.orchestra.run.smithi196:> dd of=/etc/ceph/ceph.conf 2024-02-19T21:45:17.594 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-02-19T21:45:17.594 DEBUG:teuthology.orchestra.run.smithi196:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:45:17.655 INFO:tasks.cephadm:Adding host smithi196 to orchestrator... 2024-02-19T21:45:17.655 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch host add smithi196 2024-02-19T21:45:18.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:17.011754+0000 mgr.smithi131.cjubsp (mgr.14164) 6 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:45:18.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:17.014270+0000 mon.smithi131 (mon.0) 117 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:24.704697+0000 mgr.smithi131.cjubsp (mgr.14164) 7 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:28.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:26.705122+0000 mgr.smithi131.cjubsp (mgr.14164) 8 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:29.909 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:28.705407+0000 mgr.smithi131.cjubsp (mgr.14164) 9 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:29.910 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:29.225908+0000 mon.smithi131 (mon.0) 118 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:29.910 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:29.227364+0000 mon.smithi131 (mon.0) 119 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi131", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:45:29.910 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:29.228928+0000 mon.smithi131 (mon.0) 120 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi131", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-02-19T21:45:29.910 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:29.229484+0000 mon.smithi131 (mon.0) 121 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:45:31.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:45:29.229968+0000 mgr.smithi131.cjubsp (mgr.14164) 10 : cephadm [INF] Deploying daemon crash.smithi131 on smithi131 2024-02-19T21:45:31.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:29.830675+0000 mon.smithi131 (mon.0) 122 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:30.706992+0000 mgr.smithi131.cjubsp (mgr.14164) 11 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:31.523581+0000 mon.smithi131 (mon.0) 123 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:31.571261+0000 mon.smithi131 (mon.0) 124 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:31.573020+0000 mon.smithi131 (mon.0) 125 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:32.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:31.573346+0000 mon.smithi131 (mon.0) 126 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-19T21:45:32.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:31.575876+0000 mon.smithi131 (mon.0) 127 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:33.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:31.573670+0000 mgr.smithi131.cjubsp (mgr.14164) 12 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-19T21:45:33.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:45:31.579160+0000 mgr.smithi131.cjubsp (mgr.14164) 13 : cephadm [INF] Deploying daemon grafana.smithi131 on smithi131 2024-02-19T21:45:34.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:32.707340+0000 mgr.smithi131.cjubsp (mgr.14164) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:34.707615+0000 mgr.smithi131.cjubsp (mgr.14164) 15 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:34.834153+0000 mon.smithi131 (mon.0) 128 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:36.708043+0000 mgr.smithi131.cjubsp (mgr.14164) 16 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:40.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:38.708323+0000 mgr.smithi131.cjubsp (mgr.14164) 17 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:40.708561+0000 mgr.smithi131.cjubsp (mgr.14164) 18 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:43.145 INFO:teuthology.orchestra.run.smithi196.stdout:Added host 'smithi196' with addr '172.21.15.196' 2024-02-19T21:45:43.747 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch host ls --format=json 2024-02-19T21:45:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:42.536717+0000 mgr.smithi131.cjubsp (mgr.14164) 19 : audit [DBG] from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi196", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:45:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:42.708865+0000 mgr.smithi131.cjubsp (mgr.14164) 20 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:43.144288+0000 mon.smithi131 (mon.0) 129 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:45.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:45:43.144631+0000 mgr.smithi131.cjubsp (mgr.14164) 21 : cephadm [INF] Added host smithi196 2024-02-19T21:45:45.629 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:45:45.629 INFO:teuthology.orchestra.run.smithi196.stdout:[{"addr": "172.21.15.131", "hostname": "smithi131", "labels": [], "status": ""}, {"addr": "172.21.15.196", "hostname": "smithi196", "labels": [], "status": ""}] 2024-02-19T21:45:46.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:44.709127+0000 mgr.smithi131.cjubsp (mgr.14164) 22 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:46.344 INFO:tasks.cephadm:Setting crush tunables to default 2024-02-19T21:45:46.344 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd crush tunables default 2024-02-19T21:45:47.029 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:45.629663+0000 mgr.smithi131.cjubsp (mgr.14164) 23 : audit [DBG] from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:45:48.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:46.709502+0000 mgr.smithi131.cjubsp (mgr.14164) 24 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:49.719 INFO:teuthology.orchestra.run.smithi131.stderr:adjusted tunables profile to default 2024-02-19T21:45:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:48.709772+0000 mgr.smithi131.cjubsp (mgr.14164) 25 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:50.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:49.018536+0000 mon.smithi131 (mon.0) 130 : audit [INF] from='client.? 172.21.15.131:0/2096969689' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-02-19T21:45:50.333 INFO:tasks.cephadm:Adding mon.smithi131 on smithi131 2024-02-19T21:45:50.333 INFO:tasks.cephadm:Adding mon.smithi196 on smithi196 2024-02-19T21:45:50.334 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch apply mon '2;smithi131:172.21.15.131=smithi131;smithi196:172.21.15.196=smithi196' 2024-02-19T21:45:50.965 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:49.715682+0000 mon.smithi131 (mon.0) 131 : audit [INF] from='client.? 172.21.15.131:0/2096969689' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-02-19T21:45:50.966 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:49.715726+0000 mon.smithi131 (mon.0) 132 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2024-02-19T21:45:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:50.710024+0000 mgr.smithi131.cjubsp (mgr.14164) 26 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:50.966125+0000 mon.smithi131 (mon.0) 133 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:45:50.968206+0000 mgr.smithi131.cjubsp (mgr.14164) 27 : cephadm [INF] Deploying daemon node-exporter.smithi131 on smithi131 2024-02-19T21:45:52.406 INFO:teuthology.orchestra.run.smithi196.stdout:Scheduled mon update... 2024-02-19T21:45:53.043 DEBUG:teuthology.orchestra.run.smithi196:mon.smithi196> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi196.service 2024-02-19T21:45:53.046 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:45:53.046 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:45:53.075 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:-- Logs begin at Mon 2024-02-19 21:36:23 UTC. -- 2024-02-19T21:45:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:52.400018+0000 mgr.smithi131.cjubsp (mgr.14164) 28 : audit [DBG] from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi131:172.21.15.131=smithi131;smithi196:172.21.15.196=smithi196", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:45:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:45:52.401247+0000 mgr.smithi131.cjubsp (mgr.14164) 29 : cephadm [INF] Saving service mon spec with placement smithi131:172.21.15.131=smithi131;smithi196:172.21.15.196=smithi196;count:2 2024-02-19T21:45:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:52.406260+0000 mon.smithi131 (mon.0) 134 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:52.710384+0000 mgr.smithi131.cjubsp (mgr.14164) 30 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:55.036 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:45:55.036 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:43:56.182677Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-19T21:45:55.037 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-02-19T21:45:56.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:54.710609+0000 mgr.smithi131.cjubsp (mgr.14164) 31 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:56.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:54.838248+0000 mon.smithi131 (mon.0) 135 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:45:56.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:55.036197+0000 mon.smithi131 (mon.0) 136 : audit [DBG] from='client.? 172.21.15.196:0/3714093364' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:45:56.666 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:45:56.666 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:45:58.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:56.710976+0000 mgr.smithi131.cjubsp (mgr.14164) 32 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:45:58.887 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:45:58.888 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:43:56.182677Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-19T21:45:58.889 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-02-19T21:46:00.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:45:58.711392+0000 mgr.smithi131.cjubsp (mgr.14164) 33 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:00.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:45:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:45:58.887927+0000 mon.smithi131 (mon.0) 137 : audit [DBG] from='client.? 172.21.15.196:0/2669867480' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:00.555 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:46:00.555 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:46:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:00.643339+0000 mon.smithi131 (mon.0) 138 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:00.649388+0000 mgr.smithi131.cjubsp (mgr.14164) 34 : cephadm [INF] Deploying daemon prometheus.smithi131 on smithi131 2024-02-19T21:46:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:00.711652+0000 mgr.smithi131.cjubsp (mgr.14164) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:03.883 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:46:03.883 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:43:56.182677Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-19T21:46:03.885 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-02-19T21:46:04.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:02.711959+0000 mgr.smithi131.cjubsp (mgr.14164) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:03.883478+0000 mon.smithi131 (mon.0) 139 : audit [DBG] from='client.? 172.21.15.196:0/2496241949' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:05.495 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:46:05.495 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:46:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:04.712297+0000 mgr.smithi131.cjubsp (mgr.14164) 37 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:04.841087+0000 mon.smithi131 (mon.0) 140 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:04.843346+0000 mon.smithi131 (mon.0) 141 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:46:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:04.845973+0000 mon.smithi131 (mon.0) 142 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:46:07.529 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:46:07.529 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:43:56.182677Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-19T21:46:07.531 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-02-19T21:46:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:06.712644+0000 mgr.smithi131.cjubsp (mgr.14164) 38 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:07.529928+0000 mon.smithi131 (mon.0) 143 : audit [DBG] from='client.? 172.21.15.196:0/1455665397' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:10.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:08.712893+0000 mgr.smithi131.cjubsp (mgr.14164) 39 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:11.312 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:46:11.313 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:46:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:10.713081+0000 mgr.smithi131.cjubsp (mgr.14164) 40 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:14.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:12.713319+0000 mgr.smithi131.cjubsp (mgr.14164) 41 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:16.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:14.713596+0000 mgr.smithi131.cjubsp (mgr.14164) 42 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:16.713967+0000 mgr.smithi131.cjubsp (mgr.14164) 43 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:17.518182+0000 mon.smithi131 (mon.0) 144 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:17.519117+0000 mon.smithi131 (mon.0) 145 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:46:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:17.520103+0000 mon.smithi131 (mon.0) 146 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:17.520798+0000 mon.smithi131 (mon.0) 147 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:46:19.118 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:46:19.118 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:43:56.182677Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-19T21:46:19.120 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-02-19T21:46:20.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:18.714295+0000 mgr.smithi131.cjubsp (mgr.14164) 44 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:20.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:18.983652+0000 mon.smithi131 (mon.0) 148 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:20.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:19.118892+0000 mon.smithi131 (mon.0) 149 : audit [DBG] from='client.? 172.21.15.196:0/1594226360' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:20.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:19.384864+0000 mon.smithi131 (mon.0) 150 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:20.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:19.845069+0000 mon.smithi131 (mon.0) 151 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:20.789 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:46:20.789 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:46:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:20.714569+0000 mgr.smithi131.cjubsp (mgr.14164) 45 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:23.385 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:46:23.385 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:43:56.182677Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-19T21:46:23.386 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-02-19T21:46:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:22.714854+0000 mgr.smithi131.cjubsp (mgr.14164) 46 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:23.385404+0000 mon.smithi131 (mon.0) 152 : audit [DBG] from='client.? 172.21.15.196:0/3732553586' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:23.975903+0000 mon.smithi131 (mon.0) 153 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:25.363 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:46:25.363 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:46:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:24.634161+0000 mon.smithi131 (mon.0) 154 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:24.715078+0000 mgr.smithi131.cjubsp (mgr.14164) 47 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:24.879088+0000 mgr.smithi131.cjubsp (mgr.14164) 48 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.conf 2024-02-19T21:46:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:24.900018+0000 mgr.smithi131.cjubsp (mgr.14164) 49 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:46:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:24.928528+0000 mon.smithi131 (mon.0) 155 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:25.160632+0000 mon.smithi131 (mon.0) 156 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:25.161127+0000 mon.smithi131 (mon.0) 157 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi196", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:46:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:25.163688+0000 mon.smithi131 (mon.0) 158 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:25.165079+0000 mon.smithi131 (mon.0) 159 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi196", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:46:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:25.166751+0000 mon.smithi131 (mon.0) 160 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi196", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-02-19T21:46:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:25.167300+0000 mon.smithi131 (mon.0) 161 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:27.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:25.167928+0000 mgr.smithi131.cjubsp (mgr.14164) 50 : cephadm [INF] Deploying daemon crash.smithi196 on smithi196 2024-02-19T21:46:27.853 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:46:27.854 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:43:56.182677Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-19T21:46:27.854 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-02-19T21:46:28.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:26.715355+0000 mgr.smithi131.cjubsp (mgr.14164) 51 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:29.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:27.758997+0000 mon.smithi131 (mon.0) 162 : audit [DBG] from='client.? 172.21.15.196:0/2048629239' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:29.572 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:46:29.572 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:46:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:28.715724+0000 mgr.smithi131.cjubsp (mgr.14164) 52 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:29.234739+0000 mon.smithi131 (mon.0) 163 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:29.236942+0000 mon.smithi131 (mon.0) 164 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi196.voprex", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:46:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:29.238923+0000 mon.smithi131 (mon.0) 165 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi196.voprex", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-02-19T21:46:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:29.239699+0000 mon.smithi131 (mon.0) 166 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:46:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:29.240607+0000 mon.smithi131 (mon.0) 167 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:29.241423+0000 mgr.smithi131.cjubsp (mgr.14164) 53 : cephadm [INF] Deploying daemon mgr.smithi196.voprex on smithi196 2024-02-19T21:46:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:29.849662+0000 mon.smithi131 (mon.0) 168 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:30.716107+0000 mgr.smithi131.cjubsp (mgr.14164) 54 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:32.822 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:46:32.823 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:43:56.182677Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-19T21:46:32.824 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-02-19T21:46:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:32.434413+0000 mon.smithi131 (mon.0) 169 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:32.436372+0000 mon.smithi131 (mon.0) 170 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:46:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:32.437199+0000 mon.smithi131 (mon.0) 171 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:32.437973+0000 mgr.smithi131.cjubsp (mgr.14164) 55 : cephadm [INF] Deploying daemon mon.smithi196 on smithi196 2024-02-19T21:46:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:32.716416+0000 mgr.smithi131.cjubsp (mgr.14164) 56 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:32.823373+0000 mon.smithi131 (mon.0) 172 : audit [DBG] from='client.? 172.21.15.196:0/147145647' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:35.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:35 smithi196 systemd[1]: Starting Ceph mon.smithi196 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:46:36.060 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-19T21:46:36.060 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mon dump -f json 2024-02-19T21:46:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:34.716745+0000 mgr.smithi131.cjubsp (mgr.14164) 57 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:34.853030+0000 mon.smithi131 (mon.0) 173 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.103+0000 7fcf7991c700 0 set uid:gid to 167:167 (ceph:ceph) 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.103+0000 7fcf7991c700 0 ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable), process ceph-mon, pid 7 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.103+0000 7fcf7991c700 0 pidfile_write: ignore empty --pid-file 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.108+0000 7fcf7991c700 0 load: jerasure load: lrc load: isa 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: RocksDB version: 6.8.1 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Compile date Jul 8 2021 2024-02-19T21:46:36.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: DB SUMMARY 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: CURRENT file: CURRENT 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: IDENTITY file: IDENTITY 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi196/store.db dir, Total Num: 0, files: 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi196/store.db: 000003.log size: 511 ; 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.error_if_exists: 0 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.create_if_missing: 0 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.paranoid_checks: 1 2024-02-19T21:46:36.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.env: 0x562f9db3a1c0 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.fs: Posix File System 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.info_log: 0x562f9fd23a40 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_file_opening_threads: 16 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.statistics: (nil) 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.use_fsync: 0 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_log_file_size: 0 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.log_file_time_to_roll: 0 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.keep_log_file_num: 1000 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.recycle_log_file_num: 0 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.allow_fallocate: 1 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.allow_mmap_reads: 0 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.allow_mmap_writes: 0 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.use_direct_reads: 0 2024-02-19T21:46:36.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.create_missing_column_families: 0 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.db_log_dir: 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi196/store.db 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.table_cache_numshardbits: 6 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_subcompactions: 1 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_background_flushes: -1 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.advise_random_on_open: 1 2024-02-19T21:46:36.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.db_write_buffer_size: 0 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.write_buffer_manager: 0x562f9fd2bfb0 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.use_adaptive_mutex: 0 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.rate_limiter: (nil) 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.wal_recovery_mode: 2 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.enable_thread_tracking: 0 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.enable_pipelined_write: 0 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.unordered_write: 0 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-19T21:46:36.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.row_cache: None 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.wal_filter: None 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.allow_ingest_behind: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.preserve_deletes: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.two_write_queues: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.manual_wal_flush: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.atomic_flush: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.persist_stats_to_disk: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.log_readahead_size: 0 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_background_jobs: 2 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_background_compactions: -1 2024-02-19T21:46:36.430 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.delayed_write_rate : 16777216 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_total_wal_size: 0 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.stats_dump_period_sec: 600 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.stats_persist_period_sec: 600 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_open_files: -1 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bytes_per_sync: 0 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_readahead_size: 0 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Compression algorithms supported: 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-19T21:46:36.431 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: kZSTD supported: 0 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: kXpressCompression supported: 0 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: kLZ4HCCompression supported: 1 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: kLZ4Compression supported: 1 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: kBZip2Compression supported: 0 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: kZlibCompression supported: 1 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: kSnappyCompression supported: 1 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi196/store.db/MANIFEST-000001 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.merge_operator: 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_filter: None 2024-02-19T21:46:36.432 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_filter_factory: None 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.table_factory: BlockBasedTable 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562f9fc3c170) 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cache_index_and_filter_blocks: 1 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: pin_top_level_index_and_filter: 1 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: index_type: 0 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: data_block_index_type: 0 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: index_shortening: 1 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: data_block_hash_table_util_ratio: 0.750000 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: hash_index_allow_collision: 1 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: checksum: 1 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: no_block_cache: 0 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: block_cache: 0x562f9fc72f10 2024-02-19T21:46:36.433 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: block_cache_name: BinnedLRUCache 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: block_cache_options: 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: capacity : 536870912 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: num_shard_bits : 4 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: strict_capacity_limit : 0 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: high_pri_pool_ratio: 0.000 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: block_cache_compressed: (nil) 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: persistent_cache: (nil) 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: block_size: 4096 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: block_size_deviation: 10 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: block_restart_interval: 16 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: index_block_restart_interval: 1 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: metadata_block_size: 4096 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: partition_filters: 0 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: use_delta_encoding: 1 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: whole_key_filtering: 1 2024-02-19T21:46:36.434 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: verify_compression: 0 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: read_amp_bytes_per_bit: 0 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: format_version: 2 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: enable_index_compression: 1 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: block_align: 0 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.write_buffer_size: 33554432 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_write_buffer_number: 2 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compression: NoCompression 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bottommost_compression: Disabled 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.prefix_extractor: nullptr 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.num_levels: 7 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-19T21:46:36.435 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compression_opts.window_bits: -14 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compression_opts.level: 32767 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compression_opts.strategy: 0 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compression_opts.enabled: false 2024-02-19T21:46:36.436 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.target_file_size_base: 67108864 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.target_file_size_multiplier: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-19T21:46:36.437 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.arena_block_size: 4194304 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.disable_auto_compactions: 0 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-19T21:46:36.438 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.table_properties_collectors: 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.inplace_update_support: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.memtable_huge_page_size: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.bloom_locality: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.max_successive_merges: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.paranoid_file_checks: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.force_consistency_checks: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.report_bg_io_stats: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.ttl: 2592000 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi196/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-19T21:46:36.439 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.440 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2024-02-19T21:46:36.440 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.440 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379196110967, "job": 1, "event": "recovery_started", "log_files": [3]} 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.109+0000 7fcf7991c700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.110+0000 7fcf7991c700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.110+0000 7fcf7991c700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379196111446, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1708379196, "oldest_key_time": 3, "file_creation_time": 0}} 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.110+0000 7fcf7991c700 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.110+0000 7fcf7991c700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379196111812, "job": 1, "event": "recovery_finished"} 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.111+0000 7fcf7991c700 4 rocksdb: DB pointer 0x562f9fd37800 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.111+0000 7fcf60724700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.111+0000 7fcf60724700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ** DB Stats ** 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-19T21:46:36.441 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ** Compaction Stats [default] ** 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ** Compaction Stats [default] ** 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:46:36.442 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Flush(GB): cumulative 0.000, interval 0.000 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ** Compaction Stats [default] ** 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2024-02-19T21:46:36.443 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ** Compaction Stats [default] ** 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Flush(GB): cumulative 0.000, interval 0.000 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:46:36.444 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.111+0000 7fcf7991c700 0 mon.smithi196 does not exist in monmap, will attempt to join an existing cluster 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.111+0000 7fcf7991c700 0 using public_addr v2:172.21.15.196:0/0 -> [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.112+0000 7fcf7991c700 0 starting mon.smithi196 rank -1 at public addrs [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] at bind addrs [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi196 fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.113+0000 7fcf7991c700 1 mon.smithi196@-1(???) e0 preinit fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.141+0000 7fcf62f29700 0 mon.smithi196@-1(synchronizing).mds e1 new map 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.141+0000 7fcf62f29700 0 mon.smithi196@-1(synchronizing).mds e1 print_map 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: e1 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: enable_multiple, ever_enabled_multiple: 1,1 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: legacy client fscid: -1 2024-02-19T21:46:36.445 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: No filesystems configured 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.141+0000 7fcf62f29700 1 mon.smithi196@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.142+0000 7fcf62f29700 1 mon.smithi196@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.142+0000 7fcf62f29700 1 mon.smithi196@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.142+0000 7fcf62f29700 1 mon.smithi196@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.142+0000 7fcf62f29700 1 mon.smithi196@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.142+0000 7fcf62f29700 1 mon.smithi196@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.142+0000 7fcf62f29700 1 mon.smithi196@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.144+0000 7fcf62f29700 0 mon.smithi196@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.144+0000 7fcf62f29700 0 mon.smithi196@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.144+0000 7fcf62f29700 0 mon.smithi196@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.144+0000 7fcf62f29700 0 mon.smithi196@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-02-19T21:46:36.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:34.716745+0000 mgr.smithi131.cjubsp (mgr.14164) 57 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:36.447 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:34.853030+0000 mon.smithi131 (mon.0) 173 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:36.447 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:46:36.144+0000 7fcf62f29700 1 mon.smithi196@-1(synchronizing).paxosservice(auth 1..7) refresh upgraded, format 0 -> 3 2024-02-19T21:46:36.447 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 bash[36631]: 6a272417d66da398131aa1ac01f261ad933d98ea8f6d2e2bc5646c0f21b70d92 2024-02-19T21:46:36.447 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:36 smithi196 systemd[1]: Started Ceph mon.smithi196 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:46:41.434 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-02-19T21:46:41.434 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":2,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","modified":"2024-02-19T21:46:36.149466Z","created":"2024-02-19T21:43:56.182677Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi131","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:3300","nonce":0},{"type":"v1","addr":"172.21.15.131:6789","nonce":0}]},"addr":"172.21.15.131:6789/0","public_addr":"172.21.15.131:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi196","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:3300","nonce":0},{"type":"v1","addr":"172.21.15.196:6789","nonce":0}]},"addr":"172.21.15.196:6789/0","public_addr":"172.21.15.196:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-02-19T21:46:41.436 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 2 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:36.152122+0000 mon.smithi131 (mon.0) 175 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:36.152307+0000 mon.smithi131 (mon.0) 176 : cluster [INF] mon.smithi131 calling monitor election 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:36.164691+0000 mon.smithi131 (mon.0) 177 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:36.717164+0000 mgr.smithi131.cjubsp (mgr.14164) 58 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:37.149019+0000 mon.smithi131 (mon.0) 178 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:37.294874+0000 mon.smithi131 (mon.0) 179 : audit [DBG] from='mgr.? 172.21.15.196:0/1595876725' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/crt"}]: dispatch 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:38.149213+0000 mon.smithi131 (mon.0) 180 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:38.150196+0000 mon.smithi196 (mon.1) 1 : cluster [INF] mon.smithi196 calling monitor election 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:38.717592+0000 mgr.smithi131.cjubsp (mgr.14164) 59 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:39.149408+0000 mon.smithi131 (mon.0) 181 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:40.149812+0000 mon.smithi131 (mon.0) 182 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:40.717976+0000 mgr.smithi131.cjubsp (mgr.14164) 60 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:41.149640+0000 mon.smithi131 (mon.0) 183 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:41.178376+0000 mon.smithi131 (mon.0) 184 : cluster [INF] mon.smithi131 is new leader, mons smithi131,smithi196 in quorum (ranks 0,1) 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:41.186741+0000 mon.smithi131 (mon.0) 185 : cluster [DBG] monmap e2: 2 mons at {smithi131=[v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0],smithi196=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:41.186802+0000 mon.smithi131 (mon.0) 186 : cluster [DBG] fsmap 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:41.186841+0000 mon.smithi131 (mon.0) 187 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:41.187116+0000 mon.smithi131 (mon.0) 188 : cluster [DBG] mgrmap e17: smithi131.cjubsp(active, since 97s) 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:41.188313+0000 mon.smithi131 (mon.0) 189 : cluster [DBG] Standby manager daemon smithi196.voprex started 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:41.189739+0000 mon.smithi131 (mon.0) 190 : audit [DBG] from='mgr.? 172.21.15.196:0/1595876725' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:46:41.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:41.192361+0000 mon.smithi131 (mon.0) 191 : cluster [INF] overall HEALTH_OK 2024-02-19T21:46:41.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:41.192699+0000 mon.smithi131 (mon.0) 192 : audit [DBG] from='mgr.? 172.21.15.196:0/1595876725' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/key"}]: dispatch 2024-02-19T21:46:41.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:41.193390+0000 mon.smithi131 (mon.0) 193 : audit [DBG] from='mgr.? 172.21.15.196:0/1595876725' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:46:41.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:41.196044+0000 mon.smithi131 (mon.0) 194 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:36.152122+0000 mon.smithi131 (mon.0) 175 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:36.152307+0000 mon.smithi131 (mon.0) 176 : cluster [INF] mon.smithi131 calling monitor election 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:36.164691+0000 mon.smithi131 (mon.0) 177 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:36.717164+0000 mgr.smithi131.cjubsp (mgr.14164) 58 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:37.149019+0000 mon.smithi131 (mon.0) 178 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:37.294874+0000 mon.smithi131 (mon.0) 179 : audit [DBG] from='mgr.? 172.21.15.196:0/1595876725' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/crt"}]: dispatch 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:38.149213+0000 mon.smithi131 (mon.0) 180 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:38.150196+0000 mon.smithi196 (mon.1) 1 : cluster [INF] mon.smithi196 calling monitor election 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:38.717592+0000 mgr.smithi131.cjubsp (mgr.14164) 59 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:41.526 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:39.149408+0000 mon.smithi131 (mon.0) 181 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:40.149812+0000 mon.smithi131 (mon.0) 182 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:40.717976+0000 mgr.smithi131.cjubsp (mgr.14164) 60 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:41.149640+0000 mon.smithi131 (mon.0) 183 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:41.178376+0000 mon.smithi131 (mon.0) 184 : cluster [INF] mon.smithi131 is new leader, mons smithi131,smithi196 in quorum (ranks 0,1) 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:41.186741+0000 mon.smithi131 (mon.0) 185 : cluster [DBG] monmap e2: 2 mons at {smithi131=[v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0],smithi196=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:41.186802+0000 mon.smithi131 (mon.0) 186 : cluster [DBG] fsmap 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:41.186841+0000 mon.smithi131 (mon.0) 187 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:41.187116+0000 mon.smithi131 (mon.0) 188 : cluster [DBG] mgrmap e17: smithi131.cjubsp(active, since 97s) 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:41.188313+0000 mon.smithi131 (mon.0) 189 : cluster [DBG] Standby manager daemon smithi196.voprex started 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:41.189739+0000 mon.smithi131 (mon.0) 190 : audit [DBG] from='mgr.? 172.21.15.196:0/1595876725' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:41.192361+0000 mon.smithi131 (mon.0) 191 : cluster [INF] overall HEALTH_OK 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:41.192699+0000 mon.smithi131 (mon.0) 192 : audit [DBG] from='mgr.? 172.21.15.196:0/1595876725' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/key"}]: dispatch 2024-02-19T21:46:41.527 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:41.193390+0000 mon.smithi131 (mon.0) 193 : audit [DBG] from='mgr.? 172.21.15.196:0/1595876725' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:46:41.528 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:41.196044+0000 mon.smithi131 (mon.0) 194 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:42.044 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-02-19T21:46:42.045 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph config generate-minimal-conf 2024-02-19T21:46:42.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:46:41.197623+0000 mgr.smithi131.cjubsp (mgr.14164) 61 : cephadm [INF] Deploying daemon node-exporter.smithi196 on smithi196 2024-02-19T21:46:42.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:41.249099+0000 mon.smithi131 (mon.0) 195 : cluster [DBG] mgrmap e18: smithi131.cjubsp(active, since 97s), standbys: smithi196.voprex 2024-02-19T21:46:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:41.249234+0000 mon.smithi131 (mon.0) 196 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi196.voprex", "id": "smithi196.voprex"}]: dispatch 2024-02-19T21:46:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:41.434765+0000 mon.smithi131 (mon.0) 197 : audit [DBG] from='client.? 172.21.15.196:0/3861581295' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:42.149755+0000 mon.smithi131 (mon.0) 198 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:41.197623+0000 mgr.smithi131.cjubsp (mgr.14164) 61 : cephadm [INF] Deploying daemon node-exporter.smithi196 on smithi196 2024-02-19T21:46:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:41.249099+0000 mon.smithi131 (mon.0) 195 : cluster [DBG] mgrmap e18: smithi131.cjubsp(active, since 97s), standbys: smithi196.voprex 2024-02-19T21:46:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:41.249234+0000 mon.smithi131 (mon.0) 196 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi196.voprex", "id": "smithi196.voprex"}]: dispatch 2024-02-19T21:46:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:41.434765+0000 mon.smithi131 (mon.0) 197 : audit [DBG] from='client.? 172.21.15.196:0/3861581295' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-19T21:46:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:42.149755+0000 mon.smithi131 (mon.0) 198 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:46:43.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:42.718353+0000 mgr.smithi131.cjubsp (mgr.14164) 62 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:42.718353+0000 mgr.smithi131.cjubsp (mgr.14164) 62 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:44.670 INFO:teuthology.orchestra.run.smithi131.stdout:# minimal ceph.conf for e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:46:44.670 INFO:teuthology.orchestra.run.smithi131.stdout:[global] 2024-02-19T21:46:44.670 INFO:teuthology.orchestra.run.smithi131.stdout: fsid = e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:46:44.670 INFO:teuthology.orchestra.run.smithi131.stdout: mon_host = [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] 2024-02-19T21:46:44.971 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:44.670678+0000 mon.smithi131 (mon.0) 199 : audit [DBG] from='client.? 172.21.15.131:0/2831378963' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:45.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:44.670678+0000 mon.smithi131 (mon.0) 199 : audit [DBG] from='client.? 172.21.15.131:0/2831378963' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:45.195 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-02-19T21:46:45.195 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:46:45.195 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd of=/etc/ceph/ceph.conf 2024-02-19T21:46:45.226 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:46:45.226 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:46:45.293 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-02-19T21:46:45.294 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/etc/ceph/ceph.conf 2024-02-19T21:46:45.332 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-02-19T21:46:45.333 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:46:45.404 INFO:tasks.cephadm:Deploying OSDs... 2024-02-19T21:46:45.405 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:46:45.405 DEBUG:teuthology.orchestra.run.smithi131:> dd if=/scratch_devs of=/dev/stdout 2024-02-19T21:46:45.423 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-02-19T21:46:45.423 DEBUG:teuthology.orchestra.run.smithi131:> stat /dev/vg_nvme/lv_1 2024-02-19T21:46:45.481 INFO:teuthology.orchestra.run.smithi131.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-02-19T21:46:45.489 INFO:teuthology.orchestra.run.smithi131.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-19T21:46:45.489 INFO:teuthology.orchestra.run.smithi131.stdout:Device: 6h/6d Inode: 166422 Links: 1 2024-02-19T21:46:45.489 INFO:teuthology.orchestra.run.smithi131.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-19T21:46:45.489 INFO:teuthology.orchestra.run.smithi131.stdout:Context: system_u:object_r:device_t:s0 2024-02-19T21:46:45.490 INFO:teuthology.orchestra.run.smithi131.stdout:Access: 2024-02-19 21:46:15.442701566 +0000 2024-02-19T21:46:45.490 INFO:teuthology.orchestra.run.smithi131.stdout:Modify: 2024-02-19 21:42:18.321274523 +0000 2024-02-19T21:46:45.490 INFO:teuthology.orchestra.run.smithi131.stdout:Change: 2024-02-19 21:42:18.321274523 +0000 2024-02-19T21:46:45.490 INFO:teuthology.orchestra.run.smithi131.stdout: Birth: - 2024-02-19T21:46:45.490 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-02-19T21:46:45.546 INFO:teuthology.orchestra.run.smithi131.stderr:1+0 records in 2024-02-19T21:46:45.546 INFO:teuthology.orchestra.run.smithi131.stderr:1+0 records out 2024-02-19T21:46:45.546 INFO:teuthology.orchestra.run.smithi131.stderr:512 bytes copied, 0.000259039 s, 2.0 MB/s 2024-02-19T21:46:45.547 DEBUG:teuthology.orchestra.run.smithi131:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-02-19T21:46:45.606 DEBUG:teuthology.orchestra.run.smithi131:> stat /dev/vg_nvme/lv_2 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout:Device: 6h/6d Inode: 163466 Links: 1 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout:Context: system_u:object_r:device_t:s0 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout:Access: 2024-02-19 21:46:15.442701566 +0000 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout:Modify: 2024-02-19 21:42:18.677266550 +0000 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout:Change: 2024-02-19 21:42:18.677266550 +0000 2024-02-19T21:46:45.664 INFO:teuthology.orchestra.run.smithi131.stdout: Birth: - 2024-02-19T21:46:45.664 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-02-19T21:46:45.728 INFO:teuthology.orchestra.run.smithi131.stderr:1+0 records in 2024-02-19T21:46:45.728 INFO:teuthology.orchestra.run.smithi131.stderr:1+0 records out 2024-02-19T21:46:45.728 INFO:teuthology.orchestra.run.smithi131.stderr:512 bytes copied, 0.000231216 s, 2.2 MB/s 2024-02-19T21:46:45.730 DEBUG:teuthology.orchestra.run.smithi131:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-02-19T21:46:45.789 DEBUG:teuthology.orchestra.run.smithi131:> stat /dev/vg_nvme/lv_3 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout:Device: 6h/6d Inode: 167421 Links: 1 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout:Context: system_u:object_r:device_t:s0 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout:Access: 2024-02-19 21:46:15.442701566 +0000 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout:Modify: 2024-02-19 21:42:19.015258981 +0000 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout:Change: 2024-02-19 21:42:19.015258981 +0000 2024-02-19T21:46:45.847 INFO:teuthology.orchestra.run.smithi131.stdout: Birth: - 2024-02-19T21:46:45.848 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-02-19T21:46:45.913 INFO:teuthology.orchestra.run.smithi131.stderr:1+0 records in 2024-02-19T21:46:45.913 INFO:teuthology.orchestra.run.smithi131.stderr:1+0 records out 2024-02-19T21:46:45.913 INFO:teuthology.orchestra.run.smithi131.stderr:512 bytes copied, 0.000190102 s, 2.7 MB/s 2024-02-19T21:46:45.914 DEBUG:teuthology.orchestra.run.smithi131:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-02-19T21:46:45.973 DEBUG:teuthology.orchestra.run.smithi131:> stat /dev/vg_nvme/lv_4 2024-02-19T21:46:46.031 INFO:teuthology.orchestra.run.smithi131.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-02-19T21:46:46.031 INFO:teuthology.orchestra.run.smithi131.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-19T21:46:46.031 INFO:teuthology.orchestra.run.smithi131.stdout:Device: 6h/6d Inode: 163546 Links: 1 2024-02-19T21:46:46.031 INFO:teuthology.orchestra.run.smithi131.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-19T21:46:46.031 INFO:teuthology.orchestra.run.smithi131.stdout:Context: system_u:object_r:device_t:s0 2024-02-19T21:46:46.031 INFO:teuthology.orchestra.run.smithi131.stdout:Access: 2024-02-19 21:46:15.442701566 +0000 2024-02-19T21:46:46.032 INFO:teuthology.orchestra.run.smithi131.stdout:Modify: 2024-02-19 21:42:19.382250763 +0000 2024-02-19T21:46:46.032 INFO:teuthology.orchestra.run.smithi131.stdout:Change: 2024-02-19 21:42:19.382250763 +0000 2024-02-19T21:46:46.032 INFO:teuthology.orchestra.run.smithi131.stdout: Birth: - 2024-02-19T21:46:46.032 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-02-19T21:46:46.097 INFO:teuthology.orchestra.run.smithi131.stderr:1+0 records in 2024-02-19T21:46:46.097 INFO:teuthology.orchestra.run.smithi131.stderr:1+0 records out 2024-02-19T21:46:46.097 INFO:teuthology.orchestra.run.smithi131.stderr:512 bytes copied, 0.000233826 s, 2.2 MB/s 2024-02-19T21:46:46.098 DEBUG:teuthology.orchestra.run.smithi131:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-02-19T21:46:46.157 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-02-19T21:46:46.157 DEBUG:teuthology.orchestra.run.smithi196:> dd if=/scratch_devs of=/dev/stdout 2024-02-19T21:46:46.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:44.718648+0000 mgr.smithi131.cjubsp (mgr.14164) 63 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:46.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:44.858405+0000 mon.smithi131 (mon.0) 200 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:46.179 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-02-19T21:46:46.180 DEBUG:teuthology.orchestra.run.smithi196:> stat /dev/vg_nvme/lv_1 2024-02-19T21:46:46.241 INFO:teuthology.orchestra.run.smithi196.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-02-19T21:46:46.241 INFO:teuthology.orchestra.run.smithi196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-19T21:46:46.241 INFO:teuthology.orchestra.run.smithi196.stdout:Device: 6h/6d Inode: 162663 Links: 1 2024-02-19T21:46:46.241 INFO:teuthology.orchestra.run.smithi196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-19T21:46:46.241 INFO:teuthology.orchestra.run.smithi196.stdout:Context: system_u:object_r:device_t:s0 2024-02-19T21:46:46.241 INFO:teuthology.orchestra.run.smithi196.stdout:Access: 2024-02-19 21:46:41.610426301 +0000 2024-02-19T21:46:46.241 INFO:teuthology.orchestra.run.smithi196.stdout:Modify: 2024-02-19 21:42:21.164349105 +0000 2024-02-19T21:46:46.241 INFO:teuthology.orchestra.run.smithi196.stdout:Change: 2024-02-19 21:42:21.164349105 +0000 2024-02-19T21:46:46.242 INFO:teuthology.orchestra.run.smithi196.stdout: Birth: - 2024-02-19T21:46:46.242 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-02-19T21:46:46.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:44.718648+0000 mgr.smithi131.cjubsp (mgr.14164) 63 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:46.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:44.858405+0000 mon.smithi131 (mon.0) 200 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:46.310 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records in 2024-02-19T21:46:46.310 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records out 2024-02-19T21:46:46.310 INFO:teuthology.orchestra.run.smithi196.stderr:512 bytes copied, 0.000247968 s, 2.1 MB/s 2024-02-19T21:46:46.311 DEBUG:teuthology.orchestra.run.smithi196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-02-19T21:46:46.373 DEBUG:teuthology.orchestra.run.smithi196:> stat /dev/vg_nvme/lv_2 2024-02-19T21:46:46.432 INFO:teuthology.orchestra.run.smithi196.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-02-19T21:46:46.433 INFO:teuthology.orchestra.run.smithi196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-19T21:46:46.433 INFO:teuthology.orchestra.run.smithi196.stdout:Device: 6h/6d Inode: 162715 Links: 1 2024-02-19T21:46:46.433 INFO:teuthology.orchestra.run.smithi196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-19T21:46:46.433 INFO:teuthology.orchestra.run.smithi196.stdout:Context: system_u:object_r:device_t:s0 2024-02-19T21:46:46.433 INFO:teuthology.orchestra.run.smithi196.stdout:Access: 2024-02-19 21:46:41.610426301 +0000 2024-02-19T21:46:46.433 INFO:teuthology.orchestra.run.smithi196.stdout:Modify: 2024-02-19 21:42:21.493341768 +0000 2024-02-19T21:46:46.433 INFO:teuthology.orchestra.run.smithi196.stdout:Change: 2024-02-19 21:42:21.493341768 +0000 2024-02-19T21:46:46.433 INFO:teuthology.orchestra.run.smithi196.stdout: Birth: - 2024-02-19T21:46:46.434 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-02-19T21:46:46.500 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records in 2024-02-19T21:46:46.501 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records out 2024-02-19T21:46:46.501 INFO:teuthology.orchestra.run.smithi196.stderr:512 bytes copied, 0.000211484 s, 2.4 MB/s 2024-02-19T21:46:46.502 DEBUG:teuthology.orchestra.run.smithi196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-02-19T21:46:46.562 DEBUG:teuthology.orchestra.run.smithi196:> stat /dev/vg_nvme/lv_3 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout:Device: 6h/6d Inode: 166689 Links: 1 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout:Context: system_u:object_r:device_t:s0 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout:Access: 2024-02-19 21:46:41.610426301 +0000 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout:Modify: 2024-02-19 21:42:21.832334209 +0000 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout:Change: 2024-02-19 21:42:21.832334209 +0000 2024-02-19T21:46:46.622 INFO:teuthology.orchestra.run.smithi196.stdout: Birth: - 2024-02-19T21:46:46.623 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-02-19T21:46:46.688 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records in 2024-02-19T21:46:46.689 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records out 2024-02-19T21:46:46.689 INFO:teuthology.orchestra.run.smithi196.stderr:512 bytes copied, 0.000189895 s, 2.7 MB/s 2024-02-19T21:46:46.690 DEBUG:teuthology.orchestra.run.smithi196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-02-19T21:46:46.749 DEBUG:teuthology.orchestra.run.smithi196:> stat /dev/vg_nvme/lv_4 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout:Device: 6h/6d Inode: 165830 Links: 1 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout:Context: system_u:object_r:device_t:s0 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout:Access: 2024-02-19 21:46:41.610426301 +0000 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout:Modify: 2024-02-19 21:42:22.190326225 +0000 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout:Change: 2024-02-19 21:42:22.190326225 +0000 2024-02-19T21:46:46.809 INFO:teuthology.orchestra.run.smithi196.stdout: Birth: - 2024-02-19T21:46:46.810 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-02-19T21:46:46.875 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records in 2024-02-19T21:46:46.875 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records out 2024-02-19T21:46:46.875 INFO:teuthology.orchestra.run.smithi196.stderr:512 bytes copied, 0.000254173 s, 2.0 MB/s 2024-02-19T21:46:46.876 DEBUG:teuthology.orchestra.run.smithi196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-02-19T21:46:46.935 INFO:tasks.cephadm:Deploying osd.0 on smithi131 with /dev/vg_nvme/lv_4... 2024-02-19T21:46:46.935 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-02-19T21:46:47.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:46.718881+0000 mgr.smithi131.cjubsp (mgr.14164) 64 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:47.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:46.718881+0000 mgr.smithi131.cjubsp (mgr.14164) 64 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:49.118 INFO:teuthology.orchestra.run.smithi131.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-02-19T21:46:49.118 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-02-19T21:46:49.118 INFO:teuthology.orchestra.run.smithi131.stderr: stderr: 10+0 records in 2024-02-19T21:46:49.119 INFO:teuthology.orchestra.run.smithi131.stderr:10+0 records out 2024-02-19T21:46:49.119 INFO:teuthology.orchestra.run.smithi131.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0136947 s, 766 MB/s 2024-02-19T21:46:49.119 INFO:teuthology.orchestra.run.smithi131.stderr:--> Zapping successful for: 2024-02-19T21:46:49.729 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch daemon add osd smithi131:vg_nvme/lv_4 2024-02-19T21:46:49.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:48.719189+0000 mgr.smithi131.cjubsp (mgr.14164) 65 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:48.719189+0000 mgr.smithi131.cjubsp (mgr.14164) 65 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:50.719503+0000 mgr.smithi131.cjubsp (mgr.14164) 66 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:51.292185+0000 mon.smithi131 (mon.0) 201 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:51.293444+0000 mon.smithi131 (mon.0) 202 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:46:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:51.294511+0000 mon.smithi131 (mon.0) 203 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:51.295760+0000 mon.smithi131 (mon.0) 204 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:46:52.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:51.765741+0000 mon.smithi131 (mon.0) 205 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:46:52.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:51.767161+0000 mon.smithi131 (mon.0) 206 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:46:52.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:51.767661+0000 mon.smithi131 (mon.0) 207 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:50.719503+0000 mgr.smithi131.cjubsp (mgr.14164) 66 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:51.292185+0000 mon.smithi131 (mon.0) 201 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:51.293444+0000 mon.smithi131 (mon.0) 202 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:46:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:51.294511+0000 mon.smithi131 (mon.0) 203 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:51.295760+0000 mon.smithi131 (mon.0) 204 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:46:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:51.765741+0000 mon.smithi131 (mon.0) 205 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:46:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:51.767161+0000 mon.smithi131 (mon.0) 206 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:46:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:51.767661+0000 mon.smithi131 (mon.0) 207 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:52.998 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:51.764859+0000 mgr.smithi131.cjubsp (mgr.14164) 67 : audit [DBG] from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi131:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:46:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:51.764859+0000 mgr.smithi131.cjubsp (mgr.14164) 67 : audit [DBG] from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi131:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:46:53.867 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:52.719872+0000 mgr.smithi131.cjubsp (mgr.14164) 68 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:53.867 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:53.157796+0000 mon.smithi131 (mon.0) 208 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:53.867 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:53.211327+0000 mon.smithi131 (mon.0) 209 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:52.719872+0000 mgr.smithi131.cjubsp (mgr.14164) 68 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:53.157796+0000 mon.smithi131 (mon.0) 208 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:53.211327+0000 mon.smithi131 (mon.0) 209 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:46:53.158194+0000 mgr.smithi131.cjubsp (mgr.14164) 69 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.conf 2024-02-19T21:46:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:46:53.183049+0000 mgr.smithi131.cjubsp (mgr.14164) 70 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:46:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:54.586396+0000 mon.smithi131 (mon.0) 210 : audit [INF] from='client.? 172.21.15.131:0/1600044209' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9332188a-8d84-40a8-b382-1eca712a6a3a"}]: dispatch 2024-02-19T21:46:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:54.593529+0000 mon.smithi131 (mon.0) 211 : audit [INF] from='client.? 172.21.15.131:0/1600044209' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9332188a-8d84-40a8-b382-1eca712a6a3a"}]': finished 2024-02-19T21:46:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:54.593598+0000 mon.smithi131 (mon.0) 212 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-02-19T21:46:55.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:54.593716+0000 mon.smithi131 (mon.0) 213 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:46:55.226 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:53.158194+0000 mgr.smithi131.cjubsp (mgr.14164) 69 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.conf 2024-02-19T21:46:55.226 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:53.183049+0000 mgr.smithi131.cjubsp (mgr.14164) 70 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:46:55.227 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:54.586396+0000 mon.smithi131 (mon.0) 210 : audit [INF] from='client.? 172.21.15.131:0/1600044209' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9332188a-8d84-40a8-b382-1eca712a6a3a"}]: dispatch 2024-02-19T21:46:55.227 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:54.593529+0000 mon.smithi131 (mon.0) 211 : audit [INF] from='client.? 172.21.15.131:0/1600044209' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9332188a-8d84-40a8-b382-1eca712a6a3a"}]': finished 2024-02-19T21:46:55.227 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:54.593598+0000 mon.smithi131 (mon.0) 212 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-02-19T21:46:55.227 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:54.593716+0000 mon.smithi131 (mon.0) 213 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:46:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:54.720169+0000 mgr.smithi131.cjubsp (mgr.14164) 71 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:54.862220+0000 mon.smithi131 (mon.0) 214 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:56.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:55.033390+0000 mon.smithi131 (mon.0) 215 : audit [DBG] from='client.? 172.21.15.131:0/3704201543' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:46:56.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:54.720169+0000 mgr.smithi131.cjubsp (mgr.14164) 71 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:56.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:54.862220+0000 mon.smithi131 (mon.0) 214 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:56.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:55.033390+0000 mon.smithi131 (mon.0) 215 : audit [DBG] from='client.? 172.21.15.131:0/3704201543' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:46:57.978 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:56.720394+0000 mgr.smithi131.cjubsp (mgr.14164) 72 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:57.978 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:57.121824+0000 mon.smithi131 (mon.0) 216 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:57.978 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:57.142005+0000 mon.smithi131 (mon.0) 217 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:57.978 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:57.145498+0000 mon.smithi131 (mon.0) 218 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:57.979 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:57.148346+0000 mon.smithi131 (mon.0) 219 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:46:57.979 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:57.148866+0000 mon.smithi131 (mon.0) 220 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:46:57.979 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:57.149328+0000 mon.smithi131 (mon.0) 221 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:56.720394+0000 mgr.smithi131.cjubsp (mgr.14164) 72 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:46:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:57.121824+0000 mon.smithi131 (mon.0) 216 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:57.142005+0000 mon.smithi131 (mon.0) 217 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:57.145498+0000 mon.smithi131 (mon.0) 218 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:46:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:57.148346+0000 mon.smithi131 (mon.0) 219 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:46:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:57.148866+0000 mon.smithi131 (mon.0) 220 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:46:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:57.149328+0000 mon.smithi131 (mon.0) 221 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:46:59.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:46:57.122373+0000 mgr.smithi131.cjubsp (mgr.14164) 73 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.conf 2024-02-19T21:46:59.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:46:57.148096+0000 mgr.smithi131.cjubsp (mgr.14164) 74 : cephadm [INF] Reconfiguring mon.smithi131 (unknown last config time)... 2024-02-19T21:46:59.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:46:57.149726+0000 mgr.smithi131.cjubsp (mgr.14164) 75 : cephadm [INF] Reconfiguring daemon mon.smithi131 on smithi131 2024-02-19T21:46:59.213 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:57.122373+0000 mgr.smithi131.cjubsp (mgr.14164) 73 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.conf 2024-02-19T21:46:59.213 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:57.148096+0000 mgr.smithi131.cjubsp (mgr.14164) 74 : cephadm [INF] Reconfiguring mon.smithi131 (unknown last config time)... 2024-02-19T21:46:59.213 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:57.149726+0000 mgr.smithi131.cjubsp (mgr.14164) 75 : cephadm [INF] Reconfiguring daemon mon.smithi131 on smithi131 2024-02-19T21:47:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:46:58.720636+0000 mgr.smithi131.cjubsp (mgr.14164) 76 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:59.257430+0000 mon.smithi131 (mon.0) 222 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:59.258207+0000 mon.smithi131 (mon.0) 223 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi131.cjubsp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:47:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:59.258825+0000 mon.smithi131 (mon.0) 224 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:47:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:46:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:46:59.259362+0000 mon.smithi131 (mon.0) 225 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:46:58.720636+0000 mgr.smithi131.cjubsp (mgr.14164) 76 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:59.257430+0000 mon.smithi131 (mon.0) 222 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:59.258207+0000 mon.smithi131 (mon.0) 223 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi131.cjubsp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:47:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:59.258825+0000 mon.smithi131 (mon.0) 224 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:47:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:46:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:46:59.259362+0000 mon.smithi131 (mon.0) 225 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:46:59.257962+0000 mgr.smithi131.cjubsp (mgr.14164) 77 : cephadm [INF] Reconfiguring mgr.smithi131.cjubsp (unknown last config time)... 2024-02-19T21:47:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:46:59.259802+0000 mgr.smithi131.cjubsp (mgr.14164) 78 : cephadm [INF] Reconfiguring daemon mgr.smithi131.cjubsp on smithi131 2024-02-19T21:47:01.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:59.257962+0000 mgr.smithi131.cjubsp (mgr.14164) 77 : cephadm [INF] Reconfiguring mgr.smithi131.cjubsp (unknown last config time)... 2024-02-19T21:47:01.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:46:59.259802+0000 mgr.smithi131.cjubsp (mgr.14164) 78 : cephadm [INF] Reconfiguring daemon mgr.smithi131.cjubsp on smithi131 2024-02-19T21:47:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:00.720922+0000 mgr.smithi131.cjubsp (mgr.14164) 79 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:01.441576+0000 mon.smithi131 (mon.0) 226 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:00.720922+0000 mgr.smithi131.cjubsp (mgr.14164) 79 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:01.441576+0000 mon.smithi131 (mon.0) 226 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:01.442123+0000 mgr.smithi131.cjubsp (mgr.14164) 80 : cephadm [INF] Reconfiguring alertmanager.smithi131 (dependencies changed)... 2024-02-19T21:47:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:01.444454+0000 mgr.smithi131.cjubsp (mgr.14164) 81 : cephadm [INF] Reconfiguring daemon alertmanager.smithi131 on smithi131 2024-02-19T21:47:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:02.836371+0000 mon.smithi131 (mon.0) 227 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-19T21:47:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:02.837023+0000 mon.smithi131 (mon.0) 228 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:03.217 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:01.442123+0000 mgr.smithi131.cjubsp (mgr.14164) 80 : cephadm [INF] Reconfiguring alertmanager.smithi131 (dependencies changed)... 2024-02-19T21:47:03.217 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:01.444454+0000 mgr.smithi131.cjubsp (mgr.14164) 81 : cephadm [INF] Reconfiguring daemon alertmanager.smithi131 on smithi131 2024-02-19T21:47:03.217 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:02.836371+0000 mon.smithi131 (mon.0) 227 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-19T21:47:03.217 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:02.837023+0000 mon.smithi131 (mon.0) 228 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:02.721098+0000 mgr.smithi131.cjubsp (mgr.14164) 82 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:02.837497+0000 mgr.smithi131.cjubsp (mgr.14164) 83 : cephadm [INF] Deploying daemon osd.0 on smithi131 2024-02-19T21:47:04.241 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:02.721098+0000 mgr.smithi131.cjubsp (mgr.14164) 82 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:04.241 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:02.837497+0000 mgr.smithi131.cjubsp (mgr.14164) 83 : cephadm [INF] Deploying daemon osd.0 on smithi131 2024-02-19T21:47:04.935 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:04.844406+0000 mon.smithi131 (mon.0) 229 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:47:04.935 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:04.846736+0000 mon.smithi131 (mon.0) 230 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:47:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:04.844406+0000 mon.smithi131 (mon.0) 229 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:47:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:04.846736+0000 mon.smithi131 (mon.0) 230 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:47:06.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:04.721321+0000 mgr.smithi131.cjubsp (mgr.14164) 84 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:04.721321+0000 mgr.smithi131.cjubsp (mgr.14164) 84 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:06.721623+0000 mgr.smithi131.cjubsp (mgr.14164) 85 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:07.493050+0000 mon.smithi131 (mon.0) 231 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:07.493877+0000 mon.smithi131 (mon.0) 232 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi131", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:47:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:07.494480+0000 mon.smithi131 (mon.0) 233 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:06.721623+0000 mgr.smithi131.cjubsp (mgr.14164) 85 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:07.493050+0000 mon.smithi131 (mon.0) 231 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:07.493877+0000 mon.smithi131 (mon.0) 232 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi131", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:47:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:07.494480+0000 mon.smithi131 (mon.0) 233 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:08.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:07.493613+0000 mgr.smithi131.cjubsp (mgr.14164) 86 : cephadm [INF] Reconfiguring crash.smithi131 (monmap changed)... 2024-02-19T21:47:08.941 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:07.494901+0000 mgr.smithi131.cjubsp (mgr.14164) 87 : cephadm [INF] Reconfiguring daemon crash.smithi131 on smithi131 2024-02-19T21:47:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:07.493613+0000 mgr.smithi131.cjubsp (mgr.14164) 86 : cephadm [INF] Reconfiguring crash.smithi131 (monmap changed)... 2024-02-19T21:47:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:07.494901+0000 mgr.smithi131.cjubsp (mgr.14164) 87 : cephadm [INF] Reconfiguring daemon crash.smithi131 on smithi131 2024-02-19T21:47:10.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:08.722087+0000 mgr.smithi131.cjubsp (mgr.14164) 88 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:08.722087+0000 mgr.smithi131.cjubsp (mgr.14164) 88 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:10.815 INFO:teuthology.orchestra.run.smithi131.stdout:Created osd(s) 0 on host 'smithi131' 2024-02-19T21:47:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:10.722309+0000 mgr.smithi131.cjubsp (mgr.14164) 89 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:10.809355+0000 mon.smithi131 (mon.0) 234 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:10.813876+0000 mon.smithi131 (mon.0) 235 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:12.241 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:10.722309+0000 mgr.smithi131.cjubsp (mgr.14164) 89 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:12.242 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:10.809355+0000 mon.smithi131 (mon.0) 234 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:12.242 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:10.813876+0000 mon.smithi131 (mon.0) 235 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:12.265 DEBUG:teuthology.orchestra.run.smithi131:osd.0> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.0.service 2024-02-19T21:47:12.307 INFO:tasks.cephadm:Deploying osd.1 on smithi131 with /dev/vg_nvme/lv_3... 2024-02-19T21:47:12.308 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-02-19T21:47:12.338 INFO:journalctl@ceph.osd.0.smithi131.stdout:-- Logs begin at Mon 2024-02-19 21:36:09 UTC. -- 2024-02-19T21:47:14.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:12.713996+0000 mon.smithi131 (mon.0) 236 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:14.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:12.714757+0000 mgr.smithi131.cjubsp (mgr.14164) 90 : cephadm [INF] Reconfiguring grafana.smithi131 (dependencies changed)... 2024-02-19T21:47:14.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:12.722249+0000 mgr.smithi131.cjubsp (mgr.14164) 91 : cephadm [INF] Reconfiguring daemon grafana.smithi131 on smithi131 2024-02-19T21:47:14.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:12.722514+0000 mgr.smithi131.cjubsp (mgr.14164) 92 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:14.007 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:47:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.0[44669]: debug 2024-02-19T21:47:13.932+0000 7f1e2308e080 -1 osd.0 0 log_to_monitors {default=true} 2024-02-19T21:47:14.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:12.713996+0000 mon.smithi131 (mon.0) 236 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:14.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:12.714757+0000 mgr.smithi131.cjubsp (mgr.14164) 90 : cephadm [INF] Reconfiguring grafana.smithi131 (dependencies changed)... 2024-02-19T21:47:14.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:12.722249+0000 mgr.smithi131.cjubsp (mgr.14164) 91 : cephadm [INF] Reconfiguring daemon grafana.smithi131 on smithi131 2024-02-19T21:47:14.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:12.722514+0000 mgr.smithi131.cjubsp (mgr.14164) 92 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:14.980 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:13.936395+0000 mon.smithi131 (mon.0) 237 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-19T21:47:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:13.936395+0000 mon.smithi131 (mon.0) 237 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-19T21:47:15.729 INFO:teuthology.orchestra.run.smithi131.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-02-19T21:47:15.729 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-02-19T21:47:15.729 INFO:teuthology.orchestra.run.smithi131.stderr: stderr: 10+0 records in 2024-02-19T21:47:15.729 INFO:teuthology.orchestra.run.smithi131.stderr:10+0 records out 2024-02-19T21:47:15.729 INFO:teuthology.orchestra.run.smithi131.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113281 s, 926 MB/s 2024-02-19T21:47:15.729 INFO:teuthology.orchestra.run.smithi131.stderr:--> Zapping successful for: 2024-02-19T21:47:15.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:14.722749+0000 mgr.smithi131.cjubsp (mgr.14164) 93 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:15.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:14.728157+0000 mon.smithi131 (mon.0) 238 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-19T21:47:15.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:14.728212+0000 mon.smithi131 (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-02-19T21:47:15.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:14.728326+0000 mon.smithi131 (mon.0) 240 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:47:15.741 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:14.728635+0000 mon.smithi131 (mon.0) 241 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:47:16.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:47:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.0[44669]: debug 2024-02-19T21:47:15.740+0000 7f1e195f1700 -1 osd.0 0 waiting for initial osdmap 2024-02-19T21:47:16.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:47:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.0[44669]: debug 2024-02-19T21:47:15.743+0000 7f1e13d69700 -1 osd.0 8 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-02-19T21:47:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:14.722749+0000 mgr.smithi131.cjubsp (mgr.14164) 93 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:14.728157+0000 mon.smithi131 (mon.0) 238 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-19T21:47:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:14.728212+0000 mon.smithi131 (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-02-19T21:47:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:14.728326+0000 mon.smithi131 (mon.0) 240 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:47:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:14.728635+0000 mon.smithi131 (mon.0) 241 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:47:16.444 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch daemon add osd smithi131:vg_nvme/lv_3 2024-02-19T21:47:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:15.738632+0000 mon.smithi131 (mon.0) 242 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]': finished 2024-02-19T21:47:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:15.738678+0000 mon.smithi131 (mon.0) 243 : cluster [DBG] osdmap e8: 1 total, 0 up, 1 in 2024-02-19T21:47:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:15.738907+0000 mon.smithi131 (mon.0) 244 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:47:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:15.741945+0000 mon.smithi131 (mon.0) 245 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:47:17.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:15.738632+0000 mon.smithi131 (mon.0) 242 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]': finished 2024-02-19T21:47:17.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:15.738678+0000 mon.smithi131 (mon.0) 243 : cluster [DBG] osdmap e8: 1 total, 0 up, 1 in 2024-02-19T21:47:17.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:15.738907+0000 mon.smithi131 (mon.0) 244 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:47:17.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:15.741945+0000 mon.smithi131 (mon.0) 245 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:47:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:14.886315+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:47:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:14.886484+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:47:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:16.723140+0000 mgr.smithi131.cjubsp (mgr.14164) 94 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:16.739565+0000 mon.smithi131 (mon.0) 246 : cluster [INF] osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687] boot 2024-02-19T21:47:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:16.739590+0000 mon.smithi131 (mon.0) 247 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-02-19T21:47:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:16.739971+0000 mon.smithi131 (mon.0) 248 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:47:18.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:14.886315+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:47:18.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:14.886484+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:47:18.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:16.723140+0000 mgr.smithi131.cjubsp (mgr.14164) 94 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-19T21:47:18.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:16.739565+0000 mon.smithi131 (mon.0) 246 : cluster [INF] osd.0 [v2:172.21.15.131:6802/775149687,v1:172.21.15.131:6803/775149687] boot 2024-02-19T21:47:18.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:16.739590+0000 mon.smithi131 (mon.0) 247 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-02-19T21:47:18.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:16.739971+0000 mon.smithi131 (mon.0) 248 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:47:18.902 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:17.743661+0000 mon.smithi131 (mon.0) 249 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2024-02-19T21:47:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:17.743661+0000 mon.smithi131 (mon.0) 249 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2024-02-19T21:47:20.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:18.723436+0000 mgr.smithi131.cjubsp (mgr.14164) 95 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:20.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:19.441645+0000 mon.smithi131 (mon.0) 250 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:20.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:19.514426+0000 mon.smithi131 (mon.0) 251 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:47:20.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:19.515843+0000 mon.smithi131 (mon.0) 252 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:47:20.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:19.516339+0000 mon.smithi131 (mon.0) 253 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:18.723436+0000 mgr.smithi131.cjubsp (mgr.14164) 95 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:20.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:19.441645+0000 mon.smithi131 (mon.0) 250 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:20.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:19.514426+0000 mon.smithi131 (mon.0) 251 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:47:20.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:19.515843+0000 mon.smithi131 (mon.0) 252 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:47:20.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:19.516339+0000 mon.smithi131 (mon.0) 253 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:21.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:19.442268+0000 mgr.smithi131.cjubsp (mgr.14164) 96 : cephadm [INF] Reconfiguring prometheus.smithi131 (dependencies changed)... 2024-02-19T21:47:21.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:19.444404+0000 mgr.smithi131.cjubsp (mgr.14164) 97 : cephadm [INF] Reconfiguring daemon prometheus.smithi131 on smithi131 2024-02-19T21:47:21.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:19.513452+0000 mgr.smithi131.cjubsp (mgr.14164) 98 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi131:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:47:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:19.442268+0000 mgr.smithi131.cjubsp (mgr.14164) 96 : cephadm [INF] Reconfiguring prometheus.smithi131 (dependencies changed)... 2024-02-19T21:47:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:19.444404+0000 mgr.smithi131.cjubsp (mgr.14164) 97 : cephadm [INF] Reconfiguring daemon prometheus.smithi131 on smithi131 2024-02-19T21:47:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:19.513452+0000 mgr.smithi131.cjubsp (mgr.14164) 98 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi131:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:47:22.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:20.723776+0000 mgr.smithi131.cjubsp (mgr.14164) 99 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:20.723776+0000 mgr.smithi131.cjubsp (mgr.14164) 99 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:23.972 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:22.724125+0000 mgr.smithi131.cjubsp (mgr.14164) 100 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:22.724125+0000 mgr.smithi131.cjubsp (mgr.14164) 100 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:24.724644+0000 mgr.smithi131.cjubsp (mgr.14164) 101 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:24.851630+0000 mon.smithi131 (mon.0) 254 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:24.852091+0000 mgr.smithi131.cjubsp (mgr.14164) 102 : cephadm [INF] Reconfiguring crash.smithi196 (monmap changed)... 2024-02-19T21:47:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:24.852317+0000 mon.smithi131 (mon.0) 255 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi196", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:47:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:24.852963+0000 mon.smithi131 (mon.0) 256 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:26.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:24.853399+0000 mgr.smithi131.cjubsp (mgr.14164) 103 : cephadm [INF] Reconfiguring daemon crash.smithi196 on smithi196 2024-02-19T21:47:26.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:25.844905+0000 mon.smithi131 (mon.0) 257 : audit [INF] from='client.? 172.21.15.131:0/2417041754' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bcb5683e-6881-4a06-a14d-0dd0eca432b0"}]: dispatch 2024-02-19T21:47:26.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:25.848781+0000 mon.smithi131 (mon.0) 258 : audit [INF] from='client.? 172.21.15.131:0/2417041754' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bcb5683e-6881-4a06-a14d-0dd0eca432b0"}]': finished 2024-02-19T21:47:26.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:25.848827+0000 mon.smithi131 (mon.0) 259 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-02-19T21:47:26.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:25.848906+0000 mon.smithi131 (mon.0) 260 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:24.724644+0000 mgr.smithi131.cjubsp (mgr.14164) 101 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:24.851630+0000 mon.smithi131 (mon.0) 254 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:24.852091+0000 mgr.smithi131.cjubsp (mgr.14164) 102 : cephadm [INF] Reconfiguring crash.smithi196 (monmap changed)... 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:24.852317+0000 mon.smithi131 (mon.0) 255 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi196", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:24.852963+0000 mon.smithi131 (mon.0) 256 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:24.853399+0000 mgr.smithi131.cjubsp (mgr.14164) 103 : cephadm [INF] Reconfiguring daemon crash.smithi196 on smithi196 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:25.844905+0000 mon.smithi131 (mon.0) 257 : audit [INF] from='client.? 172.21.15.131:0/2417041754' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bcb5683e-6881-4a06-a14d-0dd0eca432b0"}]: dispatch 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:25.848781+0000 mon.smithi131 (mon.0) 258 : audit [INF] from='client.? 172.21.15.131:0/2417041754' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bcb5683e-6881-4a06-a14d-0dd0eca432b0"}]': finished 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:25.848827+0000 mon.smithi131 (mon.0) 259 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-02-19T21:47:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:25.848906+0000 mon.smithi131 (mon.0) 260 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:27.082 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:26.252111+0000 mon.smithi131 (mon.0) 261 : audit [DBG] from='client.? 172.21.15.131:0/988899386' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:47:27.082 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:26.436977+0000 mon.smithi131 (mon.0) 262 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:27.082 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:26.437653+0000 mon.smithi131 (mon.0) 263 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi196.voprex", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:47:27.083 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:26.438250+0000 mon.smithi131 (mon.0) 264 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:47:27.083 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:26.438682+0000 mon.smithi131 (mon.0) 265 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:26.252111+0000 mon.smithi131 (mon.0) 261 : audit [DBG] from='client.? 172.21.15.131:0/988899386' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:47:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:26.436977+0000 mon.smithi131 (mon.0) 262 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:26.437653+0000 mon.smithi131 (mon.0) 263 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi196.voprex", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:47:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:26.438250+0000 mon.smithi131 (mon.0) 264 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:47:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:26.438682+0000 mon.smithi131 (mon.0) 265 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:28.134 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:26.437418+0000 mgr.smithi131.cjubsp (mgr.14164) 104 : cephadm [INF] Reconfiguring mgr.smithi196.voprex (monmap changed)... 2024-02-19T21:47:28.134 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:26.439089+0000 mgr.smithi131.cjubsp (mgr.14164) 105 : cephadm [INF] Reconfiguring daemon mgr.smithi196.voprex on smithi196 2024-02-19T21:47:28.134 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:26.725018+0000 mgr.smithi131.cjubsp (mgr.14164) 106 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:28.134 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:27.730482+0000 mon.smithi131 (mon.0) 266 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:28.134 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:27.731320+0000 mon.smithi131 (mon.0) 267 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:47:28.134 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:27.732055+0000 mon.smithi131 (mon.0) 268 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:47:28.134 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:27.732717+0000 mon.smithi131 (mon.0) 269 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:26.437418+0000 mgr.smithi131.cjubsp (mgr.14164) 104 : cephadm [INF] Reconfiguring mgr.smithi196.voprex (monmap changed)... 2024-02-19T21:47:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:26.439089+0000 mgr.smithi131.cjubsp (mgr.14164) 105 : cephadm [INF] Reconfiguring daemon mgr.smithi196.voprex on smithi196 2024-02-19T21:47:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:26.725018+0000 mgr.smithi131.cjubsp (mgr.14164) 106 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:27.730482+0000 mon.smithi131 (mon.0) 266 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:27.731320+0000 mon.smithi131 (mon.0) 267 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:47:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:27.732055+0000 mon.smithi131 (mon.0) 268 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:47:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:27.732717+0000 mon.smithi131 (mon.0) 269 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:29.103 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:27.731014+0000 mgr.smithi131.cjubsp (mgr.14164) 107 : cephadm [INF] Reconfiguring mon.smithi196 (monmap changed)... 2024-02-19T21:47:29.103 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:27.733289+0000 mgr.smithi131.cjubsp (mgr.14164) 108 : cephadm [INF] Reconfiguring daemon mon.smithi196 on smithi196 2024-02-19T21:47:29.145 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:27.731014+0000 mgr.smithi131.cjubsp (mgr.14164) 107 : cephadm [INF] Reconfiguring mon.smithi196 (monmap changed)... 2024-02-19T21:47:29.145 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:27.733289+0000 mgr.smithi131.cjubsp (mgr.14164) 108 : cephadm [INF] Reconfiguring daemon mon.smithi196 on smithi196 2024-02-19T21:47:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:28.725290+0000 mgr.smithi131.cjubsp (mgr.14164) 109 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:29.186708+0000 mon.smithi131 (mon.0) 270 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:29.189093+0000 mon.smithi131 (mon.0) 271 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-19T21:47:30.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:28.725290+0000 mgr.smithi131.cjubsp (mgr.14164) 109 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:30.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:29.186708+0000 mon.smithi131 (mon.0) 270 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:30.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:29.189093+0000 mon.smithi131 (mon.0) 271 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-19T21:47:31.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:29.189412+0000 mgr.smithi131.cjubsp (mgr.14164) 110 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-19T21:47:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:29.189412+0000 mgr.smithi131.cjubsp (mgr.14164) 110 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-19T21:47:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:30.725578+0000 mgr.smithi131.cjubsp (mgr.14164) 111 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:31.728825+0000 mon.smithi131 (mon.0) 272 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-19T21:47:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:31.729440+0000 mon.smithi131 (mon.0) 273 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:32.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:30.725578+0000 mgr.smithi131.cjubsp (mgr.14164) 111 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:32.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:31.728825+0000 mon.smithi131 (mon.0) 272 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-19T21:47:32.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:31.729440+0000 mon.smithi131 (mon.0) 273 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:33.083 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:31.729960+0000 mgr.smithi131.cjubsp (mgr.14164) 112 : cephadm [INF] Deploying daemon osd.1 on smithi131 2024-02-19T21:47:33.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:31.729960+0000 mgr.smithi131.cjubsp (mgr.14164) 112 : cephadm [INF] Deploying daemon osd.1 on smithi131 2024-02-19T21:47:34.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:32.725920+0000 mgr.smithi131.cjubsp (mgr.14164) 113 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:34.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:32.725920+0000 mgr.smithi131.cjubsp (mgr.14164) 113 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:35.196 INFO:teuthology.orchestra.run.smithi131.stdout:Created osd(s) 1 on host 'smithi131' 2024-02-19T21:47:35.832 DEBUG:teuthology.orchestra.run.smithi131:osd.1> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.1.service 2024-02-19T21:47:35.834 INFO:tasks.cephadm:Deploying osd.2 on smithi131 with /dev/vg_nvme/lv_2... 2024-02-19T21:47:35.835 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-02-19T21:47:35.864 INFO:journalctl@ceph.osd.1.smithi131.stdout:-- Logs begin at Mon 2024-02-19 21:36:09 UTC. -- 2024-02-19T21:47:36.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:34.726202+0000 mgr.smithi131.cjubsp (mgr.14164) 114 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.190766+0000 mon.smithi131 (mon.0) 274 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.194595+0000 mon.smithi131 (mon.0) 275 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.204374+0000 mon.smithi131 (mon.0) 276 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.131:9093"}]: dispatch 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.209358+0000 mon.smithi131 (mon.0) 277 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.212452+0000 mon.smithi131 (mon.0) 278 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.216940+0000 mon.smithi131 (mon.0) 279 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.131:3000"}]: dispatch 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.221185+0000 mon.smithi131 (mon.0) 280 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.224719+0000 mon.smithi131 (mon.0) 281 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T21:47:36.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.228968+0000 mon.smithi131 (mon.0) 282 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.131:9095"}]: dispatch 2024-02-19T21:47:36.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.232844+0000 mon.smithi131 (mon.0) 283 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.234439+0000 mon.smithi131 (mon.0) 284 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:47:36.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.235270+0000 mon.smithi131 (mon.0) 285 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:36.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.235833+0000 mon.smithi131 (mon.0) 286 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:47:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:34.726202+0000 mgr.smithi131.cjubsp (mgr.14164) 114 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.190766+0000 mon.smithi131 (mon.0) 274 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.194595+0000 mon.smithi131 (mon.0) 275 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.204374+0000 mon.smithi131 (mon.0) 276 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.131:9093"}]: dispatch 2024-02-19T21:47:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.209358+0000 mon.smithi131 (mon.0) 277 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.212452+0000 mon.smithi131 (mon.0) 278 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-19T21:47:36.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.216940+0000 mon.smithi131 (mon.0) 279 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.131:3000"}]: dispatch 2024-02-19T21:47:36.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.221185+0000 mon.smithi131 (mon.0) 280 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.224719+0000 mon.smithi131 (mon.0) 281 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T21:47:36.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.228968+0000 mon.smithi131 (mon.0) 282 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.131:9095"}]: dispatch 2024-02-19T21:47:36.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.232844+0000 mon.smithi131 (mon.0) 283 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:36.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.234439+0000 mon.smithi131 (mon.0) 284 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:47:36.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.235270+0000 mon.smithi131 (mon.0) 285 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:36.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.235833+0000 mon.smithi131 (mon.0) 286 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:47:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.204695+0000 mgr.smithi131.cjubsp (mgr.14164) 115 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.131:9093"}]: dispatch 2024-02-19T21:47:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.212770+0000 mgr.smithi131.cjubsp (mgr.14164) 116 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-19T21:47:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.217227+0000 mgr.smithi131.cjubsp (mgr.14164) 117 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.131:3000"}]: dispatch 2024-02-19T21:47:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.225182+0000 mgr.smithi131.cjubsp (mgr.14164) 118 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T21:47:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:35.229313+0000 mgr.smithi131.cjubsp (mgr.14164) 119 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.131:9095"}]: dispatch 2024-02-19T21:47:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.204695+0000 mgr.smithi131.cjubsp (mgr.14164) 115 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.131:9093"}]: dispatch 2024-02-19T21:47:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.212770+0000 mgr.smithi131.cjubsp (mgr.14164) 116 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-19T21:47:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.217227+0000 mgr.smithi131.cjubsp (mgr.14164) 117 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.131:3000"}]: dispatch 2024-02-19T21:47:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.225182+0000 mgr.smithi131.cjubsp (mgr.14164) 118 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T21:47:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:35.229313+0000 mgr.smithi131.cjubsp (mgr.14164) 119 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.131:9095"}]: dispatch 2024-02-19T21:47:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:36.726450+0000 mgr.smithi131.cjubsp (mgr.14164) 120 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:37.108421+0000 mon.smithi131 (mon.0) 287 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:38.217 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:36.726450+0000 mgr.smithi131.cjubsp (mgr.14164) 120 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:38.218 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:37.108421+0000 mon.smithi131 (mon.0) 287 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:38.506 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:47:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1[47703]: debug 2024-02-19T21:47:38.312+0000 7fb9da878080 -1 osd.1 0 log_to_monitors {default=true} 2024-02-19T21:47:38.646 INFO:teuthology.orchestra.run.smithi131.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-02-19T21:47:38.646 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-02-19T21:47:38.646 INFO:teuthology.orchestra.run.smithi131.stderr: stderr: 10+0 records in 2024-02-19T21:47:38.646 INFO:teuthology.orchestra.run.smithi131.stderr:10+0 records out 2024-02-19T21:47:38.646 INFO:teuthology.orchestra.run.smithi131.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0118263 s, 887 MB/s 2024-02-19T21:47:38.646 INFO:teuthology.orchestra.run.smithi131.stderr: stderr: 2024-02-19T21:47:38.646 INFO:teuthology.orchestra.run.smithi131.stderr:--> Zapping successful for: 2024-02-19T21:47:39.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:38.317286+0000 mon.smithi131 (mon.0) 288 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-19T21:47:39.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:38.317286+0000 mon.smithi131 (mon.0) 288 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-19T21:47:39.333 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch daemon add osd smithi131:vg_nvme/lv_2 2024-02-19T21:47:40.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:38.726875+0000 mgr.smithi131.cjubsp (mgr.14164) 121 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:40.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:38.892252+0000 mon.smithi131 (mon.0) 289 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-19T21:47:40.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:38.892295+0000 mon.smithi131 (mon.0) 290 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-02-19T21:47:40.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:38.892404+0000 mon.smithi131 (mon.0) 291 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:40.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:38.892734+0000 mon.smithi131 (mon.0) 292 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:47:40.007 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:47:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1[47703]: debug 2024-02-19T21:47:39.895+0000 7fb9d0ddb700 -1 osd.1 0 waiting for initial osdmap 2024-02-19T21:47:40.007 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:47:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1[47703]: debug 2024-02-19T21:47:39.899+0000 7fb9c954f700 -1 osd.1 13 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-02-19T21:47:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:38.726875+0000 mgr.smithi131.cjubsp (mgr.14164) 121 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:38.892252+0000 mon.smithi131 (mon.0) 289 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-19T21:47:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:38.892295+0000 mon.smithi131 (mon.0) 290 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-02-19T21:47:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:38.892404+0000 mon.smithi131 (mon.0) 291 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:40.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:38.892734+0000 mon.smithi131 (mon.0) 292 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:47:41.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:39.893755+0000 mon.smithi131 (mon.0) 293 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]': finished 2024-02-19T21:47:41.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:39.893808+0000 mon.smithi131 (mon.0) 294 : cluster [DBG] osdmap e13: 2 total, 1 up, 2 in 2024-02-19T21:47:41.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:39.894126+0000 mon.smithi131 (mon.0) 295 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:41.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:39.898536+0000 mon.smithi131 (mon.0) 296 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:41.223 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:39.893755+0000 mon.smithi131 (mon.0) 293 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]': finished 2024-02-19T21:47:41.223 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:39.893808+0000 mon.smithi131 (mon.0) 294 : cluster [DBG] osdmap e13: 2 total, 1 up, 2 in 2024-02-19T21:47:41.224 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:39.894126+0000 mon.smithi131 (mon.0) 295 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:41.224 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:39.898536+0000 mon.smithi131 (mon.0) 296 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:39.275318+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:47:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:39.275410+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:47:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:40.727228+0000 mgr.smithi131.cjubsp (mgr.14164) 122 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:40.898470+0000 mon.smithi131 (mon.0) 297 : cluster [INF] osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838] boot 2024-02-19T21:47:42.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:40.898504+0000 mon.smithi131 (mon.0) 298 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-02-19T21:47:42.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:40.898984+0000 mon.smithi131 (mon.0) 299 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:42.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:41.819352+0000 mon.smithi131 (mon.0) 300 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:47:42.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:41.820597+0000 mon.smithi131 (mon.0) 301 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:47:42.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:41.821104+0000 mon.smithi131 (mon.0) 302 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:39.275318+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:39.275410+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:40.727228+0000 mgr.smithi131.cjubsp (mgr.14164) 122 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:40.898470+0000 mon.smithi131 (mon.0) 297 : cluster [INF] osd.1 [v2:172.21.15.131:6810/3016860838,v1:172.21.15.131:6811/3016860838] boot 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:40.898504+0000 mon.smithi131 (mon.0) 298 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:40.898984+0000 mon.smithi131 (mon.0) 299 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:41.819352+0000 mon.smithi131 (mon.0) 300 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:41.820597+0000 mon.smithi131 (mon.0) 301 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:47:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:41.821104+0000 mon.smithi131 (mon.0) 302 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:43.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:41.818579+0000 mgr.smithi131.cjubsp (mgr.14164) 123 : audit [DBG] from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi131:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:47:43.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:41.907510+0000 mon.smithi131 (mon.0) 303 : cluster [DBG] osdmap e15: 2 total, 2 up, 2 in 2024-02-19T21:47:43.221 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:41.818579+0000 mgr.smithi131.cjubsp (mgr.14164) 123 : audit [DBG] from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi131:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:47:43.221 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:41.907510+0000 mon.smithi131 (mon.0) 303 : cluster [DBG] osdmap e15: 2 total, 2 up, 2 in 2024-02-19T21:47:44.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:42.727634+0000 mgr.smithi131.cjubsp (mgr.14164) 124 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:42.727634+0000 mgr.smithi131.cjubsp (mgr.14164) 124 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:45.385 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:44.069835+0000 mon.smithi131 (mon.0) 304 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:45.386 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:44.728042+0000 mgr.smithi131.cjubsp (mgr.14164) 125 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:45.386 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:44.811488+0000 mon.smithi131 (mon.0) 305 : audit [INF] from='client.? 172.21.15.131:0/1293083223' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "52e5f8fc-2bac-4ff7-9cef-fea8652db649"}]: dispatch 2024-02-19T21:47:45.386 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:44.815160+0000 mon.smithi131 (mon.0) 306 : audit [INF] from='client.? 172.21.15.131:0/1293083223' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "52e5f8fc-2bac-4ff7-9cef-fea8652db649"}]': finished 2024-02-19T21:47:45.386 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:44.815196+0000 mon.smithi131 (mon.0) 307 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-02-19T21:47:45.386 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:44.815287+0000 mon.smithi131 (mon.0) 308 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:47:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:44.069835+0000 mon.smithi131 (mon.0) 304 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:44.728042+0000 mgr.smithi131.cjubsp (mgr.14164) 125 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:44.811488+0000 mon.smithi131 (mon.0) 305 : audit [INF] from='client.? 172.21.15.131:0/1293083223' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "52e5f8fc-2bac-4ff7-9cef-fea8652db649"}]: dispatch 2024-02-19T21:47:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:44.815160+0000 mon.smithi131 (mon.0) 306 : audit [INF] from='client.? 172.21.15.131:0/1293083223' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "52e5f8fc-2bac-4ff7-9cef-fea8652db649"}]': finished 2024-02-19T21:47:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:44.815196+0000 mon.smithi131 (mon.0) 307 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-02-19T21:47:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:44.815287+0000 mon.smithi131 (mon.0) 308 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:47:46.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:45.241529+0000 mon.smithi131 (mon.0) 309 : audit [DBG] from='client.? 172.21.15.131:0/1430478519' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:47:46.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:45.241529+0000 mon.smithi131 (mon.0) 309 : audit [DBG] from='client.? 172.21.15.131:0/1430478519' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:47:47.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:46.728323+0000 mgr.smithi131.cjubsp (mgr.14164) 126 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:47.488 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:46.728323+0000 mgr.smithi131.cjubsp (mgr.14164) 126 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:50.081 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:48.728590+0000 mgr.smithi131.cjubsp (mgr.14164) 127 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:48.728590+0000 mgr.smithi131.cjubsp (mgr.14164) 127 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:51.803 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:50.592635+0000 mgr.smithi131.cjubsp (mgr.14164) 128 : cephadm [INF] Detected new or changed devices on smithi131 2024-02-19T21:47:51.803 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:50.597281+0000 mon.smithi131 (mon.0) 310 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:51.803 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:50.728877+0000 mgr.smithi131.cjubsp (mgr.14164) 129 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:51.803 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:50.952773+0000 mon.smithi131 (mon.0) 311 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:47:51.803 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:50.953360+0000 mon.smithi131 (mon.0) 312 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:47:51.803 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:50.953750+0000 mgr.smithi131.cjubsp (mgr.14164) 130 : cephadm [INF] Adjusting osd_memory_target on smithi131 to 6449M 2024-02-19T21:47:51.803 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:50.957361+0000 mon.smithi131 (mon.0) 313 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:51.804 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:50.960285+0000 mon.smithi131 (mon.0) 314 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:51.804 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:50.963018+0000 mon.smithi131 (mon.0) 315 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:47:51.804 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:50.963826+0000 mon.smithi131 (mon.0) 316 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:51.804 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:50.964327+0000 mon.smithi131 (mon.0) 317 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:47:51.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:50.592635+0000 mgr.smithi131.cjubsp (mgr.14164) 128 : cephadm [INF] Detected new or changed devices on smithi131 2024-02-19T21:47:51.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:50.597281+0000 mon.smithi131 (mon.0) 310 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:51.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:50.728877+0000 mgr.smithi131.cjubsp (mgr.14164) 129 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:51.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:50.952773+0000 mon.smithi131 (mon.0) 311 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:47:51.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:50.953360+0000 mon.smithi131 (mon.0) 312 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:47:51.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:50.953750+0000 mgr.smithi131.cjubsp (mgr.14164) 130 : cephadm [INF] Adjusting osd_memory_target on smithi131 to 6449M 2024-02-19T21:47:51.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:50.957361+0000 mon.smithi131 (mon.0) 313 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:51.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:50.960285+0000 mon.smithi131 (mon.0) 314 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:51.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:50.963018+0000 mon.smithi131 (mon.0) 315 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:47:51.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:50.963826+0000 mon.smithi131 (mon.0) 316 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:51.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:50.964327+0000 mon.smithi131 (mon.0) 317 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:47:52.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:51.823489+0000 mon.smithi131 (mon.0) 318 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-19T21:47:52.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:51.824067+0000 mon.smithi131 (mon.0) 319 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:52.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:47:51.824523+0000 mgr.smithi131.cjubsp (mgr.14164) 131 : cephadm [INF] Deploying daemon osd.2 on smithi131 2024-02-19T21:47:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:51.823489+0000 mon.smithi131 (mon.0) 318 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-19T21:47:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:51.824067+0000 mon.smithi131 (mon.0) 319 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:47:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:47:51.824523+0000 mgr.smithi131.cjubsp (mgr.14164) 131 : cephadm [INF] Deploying daemon osd.2 on smithi131 2024-02-19T21:47:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:52.729177+0000 mgr.smithi131.cjubsp (mgr.14164) 132 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:52.760560+0000 mon.smithi131 (mon.0) 320 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:54.183 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:52.729177+0000 mgr.smithi131.cjubsp (mgr.14164) 132 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:54.183 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:52.760560+0000 mon.smithi131 (mon.0) 320 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:56.072 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:54.729572+0000 mgr.smithi131.cjubsp (mgr.14164) 133 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:54.729572+0000 mgr.smithi131.cjubsp (mgr.14164) 133 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:57.561 INFO:teuthology.orchestra.run.smithi131.stdout:Created osd(s) 2 on host 'smithi131' 2024-02-19T21:47:58.170 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:56.729850+0000 mgr.smithi131.cjubsp (mgr.14164) 134 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:58.170 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:57.555708+0000 mon.smithi131 (mon.0) 321 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:58.170 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:57.560240+0000 mon.smithi131 (mon.0) 322 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:56.729850+0000 mgr.smithi131.cjubsp (mgr.14164) 134 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:47:58.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:57.555708+0000 mon.smithi131 (mon.0) 321 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:58.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:57.560240+0000 mon.smithi131 (mon.0) 322 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:47:58.764 DEBUG:teuthology.orchestra.run.smithi131:osd.2> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.2.service 2024-02-19T21:47:58.767 INFO:tasks.cephadm:Deploying osd.3 on smithi131 with /dev/vg_nvme/lv_1... 2024-02-19T21:47:58.767 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-02-19T21:47:58.801 INFO:journalctl@ceph.osd.2.smithi131.stdout:-- Logs begin at Mon 2024-02-19 21:36:09 UTC. -- 2024-02-19T21:48:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:47:58.730248+0000 mgr.smithi131.cjubsp (mgr.14164) 135 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:48:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:47:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:47:58.841165+0000 mon.smithi131 (mon.0) 323 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:47:58.730248+0000 mgr.smithi131.cjubsp (mgr.14164) 135 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:48:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:47:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:47:58.841165+0000 mon.smithi131 (mon.0) 323 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:00.688 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:48:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2[51279]: debug 2024-02-19T21:48:00.434+0000 7efc68926080 -1 osd.2 0 log_to_monitors {default=true} 2024-02-19T21:48:01.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:00.438221+0000 mon.smithi131 (mon.0) 324 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-19T21:48:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:00.438221+0000 mon.smithi131 (mon.0) 324 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-19T21:48:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:00.730594+0000 mgr.smithi131.cjubsp (mgr.14164) 136 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:48:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:00.852391+0000 mon.smithi131 (mon.0) 325 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-19T21:48:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:00.852441+0000 mon.smithi131 (mon.0) 326 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-02-19T21:48:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:00.852563+0000 mon.smithi131 (mon.0) 327 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:00.853017+0000 mon.smithi131 (mon.0) 328 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:48:02.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:00.730594+0000 mgr.smithi131.cjubsp (mgr.14164) 136 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:48:02.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:00.852391+0000 mon.smithi131 (mon.0) 325 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-19T21:48:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:00.852441+0000 mon.smithi131 (mon.0) 326 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-02-19T21:48:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:00.852563+0000 mon.smithi131 (mon.0) 327 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:00.853017+0000 mon.smithi131 (mon.0) 328 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:48:02.257 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:48:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2[51279]: debug 2024-02-19T21:48:01.856+0000 7efc5ee89700 -1 osd.2 0 waiting for initial osdmap 2024-02-19T21:48:02.257 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:48:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2[51279]: debug 2024-02-19T21:48:01.860+0000 7efc59e02700 -1 osd.2 18 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-02-19T21:48:02.588 INFO:teuthology.orchestra.run.smithi131.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-02-19T21:48:02.588 INFO:teuthology.orchestra.run.smithi131.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-02-19T21:48:02.588 INFO:teuthology.orchestra.run.smithi131.stderr: stderr: 10+0 records in 2024-02-19T21:48:02.588 INFO:teuthology.orchestra.run.smithi131.stderr:10+0 records out 2024-02-19T21:48:02.588 INFO:teuthology.orchestra.run.smithi131.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121863 s, 860 MB/s 2024-02-19T21:48:02.588 INFO:teuthology.orchestra.run.smithi131.stderr:--> Zapping successful for: 2024-02-19T21:48:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:01.854259+0000 mon.smithi131 (mon.0) 329 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]': finished 2024-02-19T21:48:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:01.854324+0000 mon.smithi131 (mon.0) 330 : cluster [DBG] osdmap e18: 3 total, 2 up, 3 in 2024-02-19T21:48:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:01.854689+0000 mon.smithi131 (mon.0) 331 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:03.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:01.858447+0000 mon.smithi131 (mon.0) 332 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:03.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:01.854259+0000 mon.smithi131 (mon.0) 329 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]': finished 2024-02-19T21:48:03.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:01.854324+0000 mon.smithi131 (mon.0) 330 : cluster [DBG] osdmap e18: 3 total, 2 up, 3 in 2024-02-19T21:48:03.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:01.854689+0000 mon.smithi131 (mon.0) 331 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:03.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:01.858447+0000 mon.smithi131 (mon.0) 332 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:03.322 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch daemon add osd smithi131:vg_nvme/lv_1 2024-02-19T21:48:03.985 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:01.408637+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:48:03.986 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:01.408747+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:48:03.986 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:02.730942+0000 mgr.smithi131.cjubsp (mgr.14164) 137 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:48:03.986 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:02.858596+0000 mon.smithi131 (mon.0) 333 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:03.986 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:02.864210+0000 mon.smithi131 (mon.0) 334 : cluster [INF] osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875] boot 2024-02-19T21:48:03.986 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:02.864244+0000 mon.smithi131 (mon.0) 335 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-02-19T21:48:03.986 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:02.864541+0000 mon.smithi131 (mon.0) 336 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:03.986 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:02.864956+0000 mon.smithi131 (mon.0) 337 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-02-19T21:48:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:01.408637+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:48:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:01.408747+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:48:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:02.730942+0000 mgr.smithi131.cjubsp (mgr.14164) 137 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-02-19T21:48:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:02.858596+0000 mon.smithi131 (mon.0) 333 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:02.864210+0000 mon.smithi131 (mon.0) 334 : cluster [INF] osd.2 [v2:172.21.15.131:6818/3047587875,v1:172.21.15.131:6819/3047587875] boot 2024-02-19T21:48:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:02.864244+0000 mon.smithi131 (mon.0) 335 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-02-19T21:48:04.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:02.864541+0000 mon.smithi131 (mon.0) 336 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:48:04.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:02.864956+0000 mon.smithi131 (mon.0) 337 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-02-19T21:48:04.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:03.869806+0000 mon.smithi131 (mon.0) 338 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2024-02-19T21:48:04.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:03.869868+0000 mon.smithi131 (mon.0) 339 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-02-19T21:48:04.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:03.871068+0000 mon.smithi131 (mon.0) 340 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-02-19T21:48:04.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:04.044242+0000 mon.smithi131 (mon.0) 341 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:04.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:04.047584+0000 mon.smithi131 (mon.0) 342 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:04.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:04.050228+0000 mon.smithi131 (mon.0) 343 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:04.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:04.050998+0000 mon.smithi131 (mon.0) 344 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:04.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:04.051416+0000 mon.smithi131 (mon.0) 345 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:04.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:04.845581+0000 mon.smithi131 (mon.0) 346 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:48:04.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:04.847486+0000 mon.smithi131 (mon.0) 347 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:03.869806+0000 mon.smithi131 (mon.0) 338 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:03.869868+0000 mon.smithi131 (mon.0) 339 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:03.871068+0000 mon.smithi131 (mon.0) 340 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:04.044242+0000 mon.smithi131 (mon.0) 341 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:04.047584+0000 mon.smithi131 (mon.0) 342 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:04.050228+0000 mon.smithi131 (mon.0) 343 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:04.050998+0000 mon.smithi131 (mon.0) 344 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:04.051416+0000 mon.smithi131 (mon.0) 345 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:04.845581+0000 mon.smithi131 (mon.0) 346 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:48:05.129 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:04.847486+0000 mon.smithi131 (mon.0) 347 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:48:06.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:04.045263+0000 mgr.smithi131.cjubsp (mgr.14164) 138 : cluster [DBG] pgmap v99: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:06.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:04.871350+0000 mon.smithi131 (mon.0) 348 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-02-19T21:48:06.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:04.871422+0000 mon.smithi131 (mon.0) 349 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-02-19T21:48:06.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:05.042519+0000 mon.smithi131 (mon.0) 350 : cluster [WRN] Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-02-19T21:48:06.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:05.577765+0000 mon.smithi131 (mon.0) 351 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:48:06.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:05.578946+0000 mon.smithi131 (mon.0) 352 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:48:06.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:05.579481+0000 mon.smithi131 (mon.0) 353 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:04.045263+0000 mgr.smithi131.cjubsp (mgr.14164) 138 : cluster [DBG] pgmap v99: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:04.871350+0000 mon.smithi131 (mon.0) 348 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-02-19T21:48:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:04.871422+0000 mon.smithi131 (mon.0) 349 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-02-19T21:48:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:05.042519+0000 mon.smithi131 (mon.0) 350 : cluster [WRN] Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-02-19T21:48:06.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:05.577765+0000 mon.smithi131 (mon.0) 351 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:48:06.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:05.578946+0000 mon.smithi131 (mon.0) 352 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:48:06.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:05.579481+0000 mon.smithi131 (mon.0) 353 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:05.576988+0000 mgr.smithi131.cjubsp (mgr.14164) 139 : audit [DBG] from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi131:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:48:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:05.917847+0000 mon.smithi131 (mon.0) 354 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:06.049047+0000 mon.smithi131 (mon.0) 355 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-02-19T21:48:07.221 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:05.576988+0000 mgr.smithi131.cjubsp (mgr.14164) 139 : audit [DBG] from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi131:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:48:07.221 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:05.917847+0000 mon.smithi131 (mon.0) 354 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:07.221 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:06.049047+0000 mon.smithi131 (mon.0) 355 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-02-19T21:48:08.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:06.045586+0000 mgr.smithi131.cjubsp (mgr.14164) 140 : cluster [DBG] pgmap v101: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:06.045586+0000 mgr.smithi131.cjubsp (mgr.14164) 140 : cluster [DBG] pgmap v101: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:08.446450+0000 mon.smithi131 (mon.0) 356 : audit [INF] from='client.? 172.21.15.131:0/4239704602' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "77028a52-76a4-49b5-9f06-58a428714aa7"}]: dispatch 2024-02-19T21:48:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:08.452994+0000 mon.smithi131 (mon.0) 357 : audit [INF] from='client.? 172.21.15.131:0/4239704602' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "77028a52-76a4-49b5-9f06-58a428714aa7"}]': finished 2024-02-19T21:48:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:08.453043+0000 mon.smithi131 (mon.0) 358 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-02-19T21:48:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:08.453182+0000 mon.smithi131 (mon.0) 359 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:08.874674+0000 mon.smithi131 (mon.0) 360 : audit [DBG] from='client.? 172.21.15.131:0/3186632115' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:48:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:08.446450+0000 mon.smithi131 (mon.0) 356 : audit [INF] from='client.? 172.21.15.131:0/4239704602' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "77028a52-76a4-49b5-9f06-58a428714aa7"}]: dispatch 2024-02-19T21:48:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:08.452994+0000 mon.smithi131 (mon.0) 357 : audit [INF] from='client.? 172.21.15.131:0/4239704602' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "77028a52-76a4-49b5-9f06-58a428714aa7"}]': finished 2024-02-19T21:48:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:08.453043+0000 mon.smithi131 (mon.0) 358 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-02-19T21:48:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:08.453182+0000 mon.smithi131 (mon.0) 359 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:08.874674+0000 mon.smithi131 (mon.0) 360 : audit [DBG] from='client.? 172.21.15.131:0/3186632115' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:48:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:08.045939+0000 mgr.smithi131.cjubsp (mgr.14164) 141 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:08.045939+0000 mgr.smithi131.cjubsp (mgr.14164) 141 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:12.167 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:10.046261+0000 mgr.smithi131.cjubsp (mgr.14164) 142 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:12.167 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:10.952032+0000 mon.smithi131 (mon.0) 361 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:10.046261+0000 mgr.smithi131.cjubsp (mgr.14164) 142 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:10.952032+0000 mon.smithi131 (mon.0) 361 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:14.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:12.046682+0000 mgr.smithi131.cjubsp (mgr.14164) 143 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:14.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:12.046682+0000 mgr.smithi131.cjubsp (mgr.14164) 143 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:16.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:14.047167+0000 mgr.smithi131.cjubsp (mgr.14164) 144 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:16.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:15.440996+0000 mon.smithi131 (mon.0) 362 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:16.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:15.443923+0000 mon.smithi131 (mon.0) 363 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:16.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:14.047167+0000 mgr.smithi131.cjubsp (mgr.14164) 144 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:16.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:15.440996+0000 mon.smithi131 (mon.0) 362 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:16.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:15.443923+0000 mon.smithi131 (mon.0) 363 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:17.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:48:15.436139+0000 mgr.smithi131.cjubsp (mgr.14164) 145 : cephadm [INF] Detected new or changed devices on smithi131 2024-02-19T21:48:17.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:15.441822+0000 mgr.smithi131.cjubsp (mgr.14164) 146 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:17.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:16.439161+0000 mon.smithi131 (mon.0) 364 : cluster [INF] Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-02-19T21:48:17.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:16.439190+0000 mon.smithi131 (mon.0) 365 : cluster [INF] Cluster is now healthy 2024-02-19T21:48:17.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:16.582051+0000 mon.smithi131 (mon.0) 366 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-19T21:48:17.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:16.582738+0000 mon.smithi131 (mon.0) 367 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:17.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:48:15.436139+0000 mgr.smithi131.cjubsp (mgr.14164) 145 : cephadm [INF] Detected new or changed devices on smithi131 2024-02-19T21:48:17.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:15.441822+0000 mgr.smithi131.cjubsp (mgr.14164) 146 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:17.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:16.439161+0000 mon.smithi131 (mon.0) 364 : cluster [INF] Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-02-19T21:48:17.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:16.439190+0000 mon.smithi131 (mon.0) 365 : cluster [INF] Cluster is now healthy 2024-02-19T21:48:17.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:16.582051+0000 mon.smithi131 (mon.0) 366 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-19T21:48:17.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:16.582738+0000 mon.smithi131 (mon.0) 367 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:18.084 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:48:16.583228+0000 mgr.smithi131.cjubsp (mgr.14164) 147 : cephadm [INF] Deploying daemon osd.3 on smithi131 2024-02-19T21:48:18.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:48:16.583228+0000 mgr.smithi131.cjubsp (mgr.14164) 147 : cephadm [INF] Deploying daemon osd.3 on smithi131 2024-02-19T21:48:19.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:17.442103+0000 mgr.smithi131.cjubsp (mgr.14164) 148 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:19.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:17.442103+0000 mgr.smithi131.cjubsp (mgr.14164) 148 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:20.373 INFO:teuthology.orchestra.run.smithi131.stdout:Created osd(s) 3 on host 'smithi131' 2024-02-19T21:48:21.070 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:19.442383+0000 mgr.smithi131.cjubsp (mgr.14164) 149 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:21.070 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:20.363054+0000 mon.smithi131 (mon.0) 368 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:21.070 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:20.364735+0000 mon.smithi131 (mon.0) 369 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:21.070 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:20.366947+0000 mon.smithi131 (mon.0) 370 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:21.070 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:20.367196+0000 mon.smithi131 (mon.0) 371 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:21.070 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:20.371458+0000 mon.smithi131 (mon.0) 372 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:21.096 DEBUG:teuthology.orchestra.run.smithi131:osd.3> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.3.service 2024-02-19T21:48:21.139 INFO:tasks.cephadm:Deploying osd.4 on smithi196 with /dev/vg_nvme/lv_4... 2024-02-19T21:48:21.139 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-02-19T21:48:21.170 INFO:journalctl@ceph.osd.3.smithi131.stdout:-- Logs begin at Mon 2024-02-19 21:36:09 UTC. -- 2024-02-19T21:48:21.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:19.442383+0000 mgr.smithi131.cjubsp (mgr.14164) 149 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:21.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:20.363054+0000 mon.smithi131 (mon.0) 368 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:21.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:20.364735+0000 mon.smithi131 (mon.0) 369 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:21.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:20.366947+0000 mon.smithi131 (mon.0) 370 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:21.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:20.367196+0000 mon.smithi131 (mon.0) 371 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:21.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:20.371458+0000 mon.smithi131 (mon.0) 372 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:23.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:21.442711+0000 mgr.smithi131.cjubsp (mgr.14164) 150 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:23.282 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-02-19T21:48:23.282 INFO:teuthology.orchestra.run.smithi196.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-02-19T21:48:23.282 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10+0 records in 2024-02-19T21:48:23.282 INFO:teuthology.orchestra.run.smithi196.stderr:10+0 records out 2024-02-19T21:48:23.282 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0142508 s, 736 MB/s 2024-02-19T21:48:23.283 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping successful for: 2024-02-19T21:48:23.316 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:21.442711+0000 mgr.smithi131.cjubsp (mgr.14164) 150 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:23.756 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:48:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3[54751]: debug 2024-02-19T21:48:23.441+0000 7fb96a09f080 -1 osd.3 0 log_to_monitors {default=true} 2024-02-19T21:48:23.922 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch daemon add osd smithi196:vg_nvme/lv_4 2024-02-19T21:48:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:23.446229+0000 mon.smithi131 (mon.0) 373 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-19T21:48:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:23.446229+0000 mon.smithi131 (mon.0) 373 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-19T21:48:25.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:23.442992+0000 mgr.smithi131.cjubsp (mgr.14164) 151 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:25.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:23.976473+0000 mon.smithi131 (mon.0) 374 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-19T21:48:25.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:23.976533+0000 mon.smithi131 (mon.0) 375 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-02-19T21:48:25.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:23.976667+0000 mon.smithi131 (mon.0) 376 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:25.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:23.976957+0000 mon.smithi131 (mon.0) 377 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:48:25.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:48:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3[54751]: debug 2024-02-19T21:48:24.980+0000 7fb960602700 -1 osd.3 0 waiting for initial osdmap 2024-02-19T21:48:25.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:48:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3[54751]: debug 2024-02-19T21:48:24.985+0000 7fb95cd7e700 -1 osd.3 25 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-02-19T21:48:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:23.442992+0000 mgr.smithi131.cjubsp (mgr.14164) 151 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:23.976473+0000 mon.smithi131 (mon.0) 374 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-19T21:48:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:23.976533+0000 mon.smithi131 (mon.0) 375 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-02-19T21:48:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:23.976667+0000 mon.smithi131 (mon.0) 376 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:23.976957+0000 mon.smithi131 (mon.0) 377 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:48:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:24.978917+0000 mon.smithi131 (mon.0) 378 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]': finished 2024-02-19T21:48:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:24.978981+0000 mon.smithi131 (mon.0) 379 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-02-19T21:48:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:24.979334+0000 mon.smithi131 (mon.0) 380 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:24.982742+0000 mon.smithi131 (mon.0) 381 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:25.267217+0000 mon.smithi131 (mon.0) 382 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:25.921160+0000 mon.smithi131 (mon.0) 383 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:48:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:25.922521+0000 mon.smithi131 (mon.0) 384 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:48:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:25.923083+0000 mon.smithi131 (mon.0) 385 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:26.376 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:24.978917+0000 mon.smithi131 (mon.0) 378 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]': finished 2024-02-19T21:48:26.376 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:24.978981+0000 mon.smithi131 (mon.0) 379 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-02-19T21:48:26.376 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:24.979334+0000 mon.smithi131 (mon.0) 380 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:26.376 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:24.982742+0000 mon.smithi131 (mon.0) 381 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:26.376 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:25.267217+0000 mon.smithi131 (mon.0) 382 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:26.377 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:25.921160+0000 mon.smithi131 (mon.0) 383 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:48:26.377 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:25.922521+0000 mon.smithi131 (mon.0) 384 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:48:26.377 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:25.923083+0000 mon.smithi131 (mon.0) 385 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:24.401595+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:48:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:24.401701+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:48:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:25.443282+0000 mgr.smithi131.cjubsp (mgr.14164) 152 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:27.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:25.920384+0000 mgr.smithi131.cjubsp (mgr.14164) 153 : audit [DBG] from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:48:27.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:25.983538+0000 mon.smithi131 (mon.0) 386 : cluster [INF] osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547] boot 2024-02-19T21:48:27.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:25.983563+0000 mon.smithi131 (mon.0) 387 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-02-19T21:48:27.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:25.983951+0000 mon.smithi131 (mon.0) 388 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:27.387 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:24.401595+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:48:27.387 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:24.401701+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:48:27.387 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:25.443282+0000 mgr.smithi131.cjubsp (mgr.14164) 152 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-19T21:48:27.387 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:25.920384+0000 mgr.smithi131.cjubsp (mgr.14164) 153 : audit [DBG] from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:48:27.387 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:25.983538+0000 mon.smithi131 (mon.0) 386 : cluster [INF] osd.3 [v2:172.21.15.131:6826/768749547,v1:172.21.15.131:6827/768749547] boot 2024-02-19T21:48:27.387 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:25.983563+0000 mon.smithi131 (mon.0) 387 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-02-19T21:48:27.387 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:25.983951+0000 mon.smithi131 (mon.0) 388 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:48:28.363 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:27.279010+0000 mon.smithi131 (mon.0) 389 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-02-19T21:48:28.363 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:27.443595+0000 mgr.smithi131.cjubsp (mgr.14164) 154 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:28.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:27.279010+0000 mon.smithi131 (mon.0) 389 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-02-19T21:48:28.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:27.443595+0000 mgr.smithi131.cjubsp (mgr.14164) 154 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:29.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.393491+0000 mon.smithi196 (mon.1) 2 : audit [INF] from='client.? 172.21.15.196:0/1958079841' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "288d638c-1abd-4b32-b4f1-339435dcf230"}]: dispatch 2024-02-19T21:48:29.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.394040+0000 mon.smithi131 (mon.0) 390 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "288d638c-1abd-4b32-b4f1-339435dcf230"}]: dispatch 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.397897+0000 mon.smithi131 (mon.0) 391 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "288d638c-1abd-4b32-b4f1-339435dcf230"}]': finished 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:28.397941+0000 mon.smithi131 (mon.0) 392 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.398023+0000 mon.smithi131 (mon.0) 393 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.816017+0000 mon.smithi196 (mon.1) 3 : audit [DBG] from='client.? 172.21.15.196:0/3536388784' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.859652+0000 mon.smithi131 (mon.0) 394 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.860539+0000 mon.smithi131 (mon.0) 395 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.861093+0000 mon.smithi131 (mon.0) 396 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.861601+0000 mon.smithi131 (mon.0) 397 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.862094+0000 mon.smithi131 (mon.0) 398 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:48:28.862501+0000 mgr.smithi131.cjubsp (mgr.14164) 155 : cephadm [INF] Adjusting osd_memory_target on smithi131 to 3224M 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.865453+0000 mon.smithi131 (mon.0) 399 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.869940+0000 mon.smithi131 (mon.0) 400 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:29.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.872615+0000 mon.smithi131 (mon.0) 401 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:29.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.873370+0000 mon.smithi131 (mon.0) 402 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:29.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:28.873846+0000 mon.smithi131 (mon.0) 403 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.393491+0000 mon.smithi196 (mon.1) 2 : audit [INF] from='client.? 172.21.15.196:0/1958079841' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "288d638c-1abd-4b32-b4f1-339435dcf230"}]: dispatch 2024-02-19T21:48:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.394040+0000 mon.smithi131 (mon.0) 390 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "288d638c-1abd-4b32-b4f1-339435dcf230"}]: dispatch 2024-02-19T21:48:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.397897+0000 mon.smithi131 (mon.0) 391 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "288d638c-1abd-4b32-b4f1-339435dcf230"}]': finished 2024-02-19T21:48:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:28.397941+0000 mon.smithi131 (mon.0) 392 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.398023+0000 mon.smithi131 (mon.0) 393 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.816017+0000 mon.smithi196 (mon.1) 3 : audit [DBG] from='client.? 172.21.15.196:0/3536388784' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.859652+0000 mon.smithi131 (mon.0) 394 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.860539+0000 mon.smithi131 (mon.0) 395 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.861093+0000 mon.smithi131 (mon.0) 396 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.861601+0000 mon.smithi131 (mon.0) 397 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.862094+0000 mon.smithi131 (mon.0) 398 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:48:28.862501+0000 mgr.smithi131.cjubsp (mgr.14164) 155 : cephadm [INF] Adjusting osd_memory_target on smithi131 to 3224M 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.865453+0000 mon.smithi131 (mon.0) 399 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.869940+0000 mon.smithi131 (mon.0) 400 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.872615+0000 mon.smithi131 (mon.0) 401 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.873370+0000 mon.smithi131 (mon.0) 402 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:29.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:28.873846+0000 mon.smithi131 (mon.0) 403 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:30.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:29.443896+0000 mgr.smithi131.cjubsp (mgr.14164) 156 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:29.443896+0000 mgr.smithi131.cjubsp (mgr.14164) 156 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:31.444161+0000 mgr.smithi131.cjubsp (mgr.14164) 157 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:32.827 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:31.444161+0000 mgr.smithi131.cjubsp (mgr.14164) 157 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:33.089012+0000 mon.smithi131 (mon.0) 404 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:34.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:33.091758+0000 mon.smithi131 (mon.0) 405 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:34.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:33.444518+0000 mgr.smithi131.cjubsp (mgr.14164) 158 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:33.089012+0000 mon.smithi131 (mon.0) 404 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:33.091758+0000 mon.smithi131 (mon.0) 405 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:33.444518+0000 mgr.smithi131.cjubsp (mgr.14164) 158 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:35.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:34.493587+0000 mon.smithi131 (mon.0) 406 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-19T21:48:35.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:34.494382+0000 mon.smithi131 (mon.0) 407 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:35.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:48:34.495013+0000 mgr.smithi131.cjubsp (mgr.14164) 159 : cephadm [INF] Deploying daemon osd.4 on smithi196 2024-02-19T21:48:35.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:34.493587+0000 mon.smithi131 (mon.0) 406 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-19T21:48:35.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:34.494382+0000 mon.smithi131 (mon.0) 407 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:35.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:48:34.495013+0000 mgr.smithi131.cjubsp (mgr.14164) 159 : cephadm [INF] Deploying daemon osd.4 on smithi196 2024-02-19T21:48:36.144 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:35.444912+0000 mgr.smithi131.cjubsp (mgr.14164) 160 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:35.444912+0000 mgr.smithi131.cjubsp (mgr.14164) 160 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:38.310 INFO:teuthology.orchestra.run.smithi196.stdout:Created osd(s) 4 on host 'smithi196' 2024-02-19T21:48:38.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:37.445479+0000 mgr.smithi131.cjubsp (mgr.14164) 161 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:38.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:38.299012+0000 mon.smithi131 (mon.0) 408 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:38.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:38.302469+0000 mon.smithi131 (mon.0) 409 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:38.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:38.302715+0000 mon.smithi131 (mon.0) 410 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:38.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:38.304190+0000 mon.smithi131 (mon.0) 411 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:38.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:38.307220+0000 mon.smithi131 (mon.0) 412 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:38.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:37.445479+0000 mgr.smithi131.cjubsp (mgr.14164) 161 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:38.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:38.299012+0000 mon.smithi131 (mon.0) 408 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:38.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:38.302469+0000 mon.smithi131 (mon.0) 409 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:38.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:38.302715+0000 mon.smithi131 (mon.0) 410 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:38.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:38.304190+0000 mon.smithi131 (mon.0) 411 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:38.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:38.307220+0000 mon.smithi131 (mon.0) 412 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:38.968 DEBUG:teuthology.orchestra.run.smithi196:osd.4> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.4.service 2024-02-19T21:48:38.970 INFO:tasks.cephadm:Deploying osd.5 on smithi196 with /dev/vg_nvme/lv_3... 2024-02-19T21:48:38.970 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-02-19T21:48:39.000 INFO:journalctl@ceph.osd.4.smithi196.stdout:-- Logs begin at Mon 2024-02-19 21:36:23 UTC. -- 2024-02-19T21:48:40.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:39.446041+0000 mgr.smithi131.cjubsp (mgr.14164) 162 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:41.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:39.446041+0000 mgr.smithi131.cjubsp (mgr.14164) 162 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:41.673 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:48:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4[40548]: debug 2024-02-19T21:48:41.432+0000 7f9bac178080 -1 osd.4 0 log_to_monitors {default=true} 2024-02-19T21:48:41.784 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-02-19T21:48:41.784 INFO:teuthology.orchestra.run.smithi196.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-02-19T21:48:41.784 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10+0 records in 2024-02-19T21:48:41.785 INFO:teuthology.orchestra.run.smithi196.stderr:10+0 records out 2024-02-19T21:48:41.785 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0120265 s, 872 MB/s 2024-02-19T21:48:41.785 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping successful for: 2024-02-19T21:48:41.913 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:41.436344+0000 mon.smithi131 (mon.0) 413 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-19T21:48:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:41.436344+0000 mon.smithi131 (mon.0) 413 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-19T21:48:42.611 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch daemon add osd smithi196:vg_nvme/lv_3 2024-02-19T21:48:42.923 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:48:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4[40548]: debug 2024-02-19T21:48:42.753+0000 7f9ba26db700 -1 osd.4 0 waiting for initial osdmap 2024-02-19T21:48:42.923 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:48:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4[40548]: debug 2024-02-19T21:48:42.759+0000 7f9b9ae4f700 -1 osd.4 30 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-02-19T21:48:42.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:41.446442+0000 mgr.smithi131.cjubsp (mgr.14164) 163 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:42.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:41.749004+0000 mon.smithi131 (mon.0) 414 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-19T21:48:42.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:41.749072+0000 mon.smithi131 (mon.0) 415 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-02-19T21:48:42.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:41.749178+0000 mon.smithi131 (mon.0) 416 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:42.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:41.749480+0000 mon.smithi131 (mon.0) 417 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:48:42.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:42.545232+0000 mon.smithi131 (mon.0) 418 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:43.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:41.446442+0000 mgr.smithi131.cjubsp (mgr.14164) 163 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:43.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:41.749004+0000 mon.smithi131 (mon.0) 414 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-19T21:48:43.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:41.749072+0000 mon.smithi131 (mon.0) 415 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-02-19T21:48:43.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:41.749178+0000 mon.smithi131 (mon.0) 416 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:43.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:41.749480+0000 mon.smithi131 (mon.0) 417 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:48:43.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:42.545232+0000 mon.smithi131 (mon.0) 418 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:43.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:42.751232+0000 mon.smithi131 (mon.0) 419 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-02-19T21:48:43.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:42.751281+0000 mon.smithi131 (mon.0) 420 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-02-19T21:48:43.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:42.751580+0000 mon.smithi131 (mon.0) 421 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:43.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:42.755852+0000 mon.smithi131 (mon.0) 422 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:43.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:43.002770+0000 mon.smithi131 (mon.0) 423 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:43.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:43.744879+0000 mon.smithi131 (mon.0) 424 : cluster [INF] osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072] boot 2024-02-19T21:48:43.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:43.744975+0000 mon.smithi131 (mon.0) 425 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-02-19T21:48:43.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:43.745496+0000 mon.smithi131 (mon.0) 426 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:42.751232+0000 mon.smithi131 (mon.0) 419 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-02-19T21:48:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:42.751281+0000 mon.smithi131 (mon.0) 420 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-02-19T21:48:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:42.751580+0000 mon.smithi131 (mon.0) 421 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:42.755852+0000 mon.smithi131 (mon.0) 422 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:43.002770+0000 mon.smithi131 (mon.0) 423 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:43.744879+0000 mon.smithi131 (mon.0) 424 : cluster [INF] osd.4 [v2:172.21.15.196:6800/10664072,v1:172.21.15.196:6801/10664072] boot 2024-02-19T21:48:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:43.744975+0000 mon.smithi131 (mon.0) 425 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-02-19T21:48:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:43.745496+0000 mon.smithi131 (mon.0) 426 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:48:45.163 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:42.396162+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:48:45.163 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:42.396246+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:48:45.163 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:43.446882+0000 mgr.smithi131.cjubsp (mgr.14164) 164 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:45.163 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:44.746272+0000 mon.smithi131 (mon.0) 427 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-02-19T21:48:45.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:42.396162+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:48:45.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:42.396246+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:48:45.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:43.446882+0000 mgr.smithi131.cjubsp (mgr.14164) 164 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-02-19T21:48:45.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:44.746272+0000 mon.smithi131 (mon.0) 427 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-02-19T21:48:46.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:45.447233+0000 mgr.smithi131.cjubsp (mgr.14164) 165 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:46.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:45.747602+0000 mon.smithi131 (mon.0) 428 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-02-19T21:48:46.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:45.803024+0000 mgr.smithi131.cjubsp (mgr.14164) 166 : audit [DBG] from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:48:46.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:45.803887+0000 mon.smithi131 (mon.0) 429 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:48:46.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:45.805551+0000 mon.smithi131 (mon.0) 430 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:48:46.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:45.806217+0000 mon.smithi131 (mon.0) 431 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:45.447233+0000 mgr.smithi131.cjubsp (mgr.14164) 165 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:45.747602+0000 mon.smithi131 (mon.0) 428 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-02-19T21:48:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:45.803024+0000 mgr.smithi131.cjubsp (mgr.14164) 166 : audit [DBG] from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:48:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:45.803887+0000 mon.smithi131 (mon.0) 429 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:48:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:45.805551+0000 mon.smithi131 (mon.0) 430 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:48:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:45.806217+0000 mon.smithi131 (mon.0) 431 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:47.447640+0000 mgr.smithi131.cjubsp (mgr.14164) 167 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:48.290321+0000 mon.smithi131 (mon.0) 432 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:48.720403+0000 mon.smithi131 (mon.0) 433 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:48.725706+0000 mon.smithi131 (mon.0) 434 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:48.731765+0000 mon.smithi131 (mon.0) 435 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:48.735490+0000 mon.smithi131 (mon.0) 436 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:48.736602+0000 mon.smithi131 (mon.0) 437 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:49.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:48.737339+0000 mon.smithi131 (mon.0) 438 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:49.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:47.447640+0000 mgr.smithi131.cjubsp (mgr.14164) 167 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:49.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:48.290321+0000 mon.smithi131 (mon.0) 432 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:49.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:48.720403+0000 mon.smithi131 (mon.0) 433 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:48:49.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:48.725706+0000 mon.smithi131 (mon.0) 434 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:49.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:48.731765+0000 mon.smithi131 (mon.0) 435 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:49.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:48.735490+0000 mon.smithi131 (mon.0) 436 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:49.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:48.736602+0000 mon.smithi131 (mon.0) 437 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:49.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:48.737339+0000 mon.smithi131 (mon.0) 438 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:48:48.285415+0000 mgr.smithi131.cjubsp (mgr.14164) 168 : cephadm [INF] Detected new or changed devices on smithi196 2024-02-19T21:48:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:48:48.721054+0000 mgr.smithi131.cjubsp (mgr.14164) 169 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 15970M 2024-02-19T21:48:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:49.159651+0000 mon.smithi196 (mon.1) 4 : audit [INF] from='client.? 172.21.15.196:0/1320254304' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "734e875d-f0f1-4204-9063-1404485397be"}]: dispatch 2024-02-19T21:48:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:49.160343+0000 mon.smithi131 (mon.0) 439 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "734e875d-f0f1-4204-9063-1404485397be"}]: dispatch 2024-02-19T21:48:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:49.169004+0000 mon.smithi131 (mon.0) 440 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "734e875d-f0f1-4204-9063-1404485397be"}]': finished 2024-02-19T21:48:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:49.169071+0000 mon.smithi131 (mon.0) 441 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-02-19T21:48:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:49.174849+0000 mon.smithi131 (mon.0) 442 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:48:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:49.572661+0000 mon.smithi196 (mon.1) 5 : audit [DBG] from='client.? 172.21.15.196:0/1981072989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:48:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:48:48.285415+0000 mgr.smithi131.cjubsp (mgr.14164) 168 : cephadm [INF] Detected new or changed devices on smithi196 2024-02-19T21:48:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:48:48.721054+0000 mgr.smithi131.cjubsp (mgr.14164) 169 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 15970M 2024-02-19T21:48:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:49.159651+0000 mon.smithi196 (mon.1) 4 : audit [INF] from='client.? 172.21.15.196:0/1320254304' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "734e875d-f0f1-4204-9063-1404485397be"}]: dispatch 2024-02-19T21:48:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:49.160343+0000 mon.smithi131 (mon.0) 439 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "734e875d-f0f1-4204-9063-1404485397be"}]: dispatch 2024-02-19T21:48:50.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:49.169004+0000 mon.smithi131 (mon.0) 440 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "734e875d-f0f1-4204-9063-1404485397be"}]': finished 2024-02-19T21:48:50.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:49.169071+0000 mon.smithi131 (mon.0) 441 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-02-19T21:48:50.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:49.174849+0000 mon.smithi131 (mon.0) 442 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:48:50.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:49.572661+0000 mon.smithi196 (mon.1) 5 : audit [DBG] from='client.? 172.21.15.196:0/1981072989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:48:51.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:49.447993+0000 mgr.smithi131.cjubsp (mgr.14164) 170 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:51.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:49.447993+0000 mgr.smithi131.cjubsp (mgr.14164) 170 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:51.448239+0000 mgr.smithi131.cjubsp (mgr.14164) 171 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:53.041 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:51.448239+0000 mgr.smithi131.cjubsp (mgr.14164) 171 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:54.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:52.990107+0000 mon.smithi131 (mon.0) 443 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:54.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:53.237573+0000 mon.smithi131 (mon.0) 444 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:54.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:52.990107+0000 mon.smithi131 (mon.0) 443 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:54.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:53.237573+0000 mon.smithi131 (mon.0) 444 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:55.065 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:53.448676+0000 mgr.smithi131.cjubsp (mgr.14164) 172 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:55.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:53.448676+0000 mgr.smithi131.cjubsp (mgr.14164) 172 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:56.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:55.084892+0000 mon.smithi131 (mon.0) 445 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-19T21:48:56.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:55.085742+0000 mon.smithi131 (mon.0) 446 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:56.307 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:55.084892+0000 mon.smithi131 (mon.0) 445 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-19T21:48:56.307 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:55.085742+0000 mon.smithi131 (mon.0) 446 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:48:55.086398+0000 mgr.smithi131.cjubsp (mgr.14164) 173 : cephadm [INF] Deploying daemon osd.5 on smithi196 2024-02-19T21:48:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:55.449179+0000 mgr.smithi131.cjubsp (mgr.14164) 174 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:48:55.086398+0000 mgr.smithi131.cjubsp (mgr.14164) 173 : cephadm [INF] Deploying daemon osd.5 on smithi196 2024-02-19T21:48:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:55.449179+0000 mgr.smithi131.cjubsp (mgr.14164) 174 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:58.797 INFO:teuthology.orchestra.run.smithi196.stdout:Created osd(s) 5 on host 'smithi196' 2024-02-19T21:48:59.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:57.449674+0000 mgr.smithi131.cjubsp (mgr.14164) 175 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:59.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:58.786339+0000 mon.smithi131 (mon.0) 447 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:59.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:58.788315+0000 mon.smithi131 (mon.0) 448 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:59.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:58.790663+0000 mon.smithi131 (mon.0) 449 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:59.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:58.790904+0000 mon.smithi131 (mon.0) 450 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:59.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:48:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:48:58.795135+0000 mon.smithi131 (mon.0) 451 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:59.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:57.449674+0000 mgr.smithi131.cjubsp (mgr.14164) 175 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:48:59.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:58.786339+0000 mon.smithi131 (mon.0) 447 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:48:59.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:58.788315+0000 mon.smithi131 (mon.0) 448 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:48:59.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:58.790663+0000 mon.smithi131 (mon.0) 449 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:59.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:58.790904+0000 mon.smithi131 (mon.0) 450 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:48:59.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:48:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:48:58.795135+0000 mon.smithi131 (mon.0) 451 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:48:59.487 DEBUG:teuthology.orchestra.run.smithi196:osd.5> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.5.service 2024-02-19T21:48:59.489 INFO:tasks.cephadm:Deploying osd.6 on smithi196 with /dev/vg_nvme/lv_2... 2024-02-19T21:48:59.489 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-02-19T21:48:59.519 INFO:journalctl@ceph.osd.5.smithi196.stdout:-- Logs begin at Mon 2024-02-19 21:36:23 UTC. -- 2024-02-19T21:49:01.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:48:59.449982+0000 mgr.smithi131.cjubsp (mgr.14164) 176 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:49:01.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:48:59.449982+0000 mgr.smithi131.cjubsp (mgr.14164) 176 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:49:02.050 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-02-19T21:49:02.050 INFO:teuthology.orchestra.run.smithi196.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-02-19T21:49:02.050 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10+0 records in 2024-02-19T21:49:02.051 INFO:teuthology.orchestra.run.smithi196.stderr:10+0 records out 2024-02-19T21:49:02.051 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121495 s, 863 MB/s 2024-02-19T21:49:02.051 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping successful for: 2024-02-19T21:49:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:01.920301+0000 mon.smithi196 (mon.1) 6 : audit [INF] from='osd.5 [v2:172.21.15.196:6808/3462679939,v1:172.21.15.196:6809/3462679939]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-19T21:49:02.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:01.920748+0000 mon.smithi131 (mon.0) 452 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-19T21:49:02.174 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:49:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.5[43428]: debug 2024-02-19T21:49:01.916+0000 7f77ffbbf080 -1 osd.5 0 log_to_monitors {default=true} 2024-02-19T21:49:02.402 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:01.920301+0000 mon.smithi196 (mon.1) 6 : audit [INF] from='osd.5 [v2:172.21.15.196:6808/3462679939,v1:172.21.15.196:6809/3462679939]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-19T21:49:02.402 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:01.920748+0000 mon.smithi131 (mon.0) 452 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-19T21:49:02.765 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch daemon add osd smithi196:vg_nvme/lv_2 2024-02-19T21:49:03.342 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:49:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.5[43428]: debug 2024-02-19T21:49:03.017+0000 7f77f6122700 -1 osd.5 0 waiting for initial osdmap 2024-02-19T21:49:03.342 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:49:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.5[43428]: debug 2024-02-19T21:49:03.021+0000 7f77ef097700 -1 osd.5 36 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-02-19T21:49:03.342 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:01.450222+0000 mgr.smithi131.cjubsp (mgr.14164) 177 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:49:03.343 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:02.013575+0000 mon.smithi131 (mon.0) 453 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-19T21:49:03.343 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:02.013621+0000 mon.smithi131 (mon.0) 454 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-02-19T21:49:03.343 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:02.013708+0000 mon.smithi131 (mon.0) 455 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:03.343 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:02.014041+0000 mon.smithi196 (mon.1) 7 : audit [INF] from='osd.5 [v2:172.21.15.196:6808/3462679939,v1:172.21.15.196:6809/3462679939]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:03.343 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:02.014420+0000 mon.smithi131 (mon.0) 456 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:01.450222+0000 mgr.smithi131.cjubsp (mgr.14164) 177 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:49:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:02.013575+0000 mon.smithi131 (mon.0) 453 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-19T21:49:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:02.013621+0000 mon.smithi131 (mon.0) 454 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-02-19T21:49:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:02.013708+0000 mon.smithi131 (mon.0) 455 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:02.014041+0000 mon.smithi196 (mon.1) 7 : audit [INF] from='osd.5 [v2:172.21.15.196:6808/3462679939,v1:172.21.15.196:6809/3462679939]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:02.014420+0000 mon.smithi131 (mon.0) 456 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:04.294 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:03.014457+0000 mon.smithi131 (mon.0) 457 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-02-19T21:49:04.295 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:03.014525+0000 mon.smithi131 (mon.0) 458 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-02-19T21:49:04.295 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:03.014916+0000 mon.smithi131 (mon.0) 459 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:04.295 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:03.018555+0000 mon.smithi131 (mon.0) 460 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:04.295 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:03.038309+0000 mon.smithi131 (mon.0) 461 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:04.295 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:03.403419+0000 mon.smithi131 (mon.0) 462 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:03.014457+0000 mon.smithi131 (mon.0) 457 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-02-19T21:49:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:03.014525+0000 mon.smithi131 (mon.0) 458 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-02-19T21:49:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:03.014916+0000 mon.smithi131 (mon.0) 459 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:03.018555+0000 mon.smithi131 (mon.0) 460 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:03.038309+0000 mon.smithi131 (mon.0) 461 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:03.403419+0000 mon.smithi131 (mon.0) 462 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:02.898657+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:49:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:02.898742+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:49:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:03.450504+0000 mgr.smithi131.cjubsp (mgr.14164) 178 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:49:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:04.020215+0000 mon.smithi131 (mon.0) 463 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:04.040876+0000 mon.smithi131 (mon.0) 464 : cluster [INF] osd.5 [v2:172.21.15.196:6808/3462679939,v1:172.21.15.196:6809/3462679939] boot 2024-02-19T21:49:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:04.040917+0000 mon.smithi131 (mon.0) 465 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-02-19T21:49:05.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:04.041055+0000 mon.smithi131 (mon.0) 466 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:05.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:04.849453+0000 mon.smithi131 (mon.0) 467 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:49:05.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:04.849654+0000 mon.smithi131 (mon.0) 468 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:49:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:02.898657+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:49:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:02.898742+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:49:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:03.450504+0000 mgr.smithi131.cjubsp (mgr.14164) 178 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-02-19T21:49:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:04.020215+0000 mon.smithi131 (mon.0) 463 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:04.040876+0000 mon.smithi131 (mon.0) 464 : cluster [INF] osd.5 [v2:172.21.15.196:6808/3462679939,v1:172.21.15.196:6809/3462679939] boot 2024-02-19T21:49:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:04.040917+0000 mon.smithi131 (mon.0) 465 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-02-19T21:49:05.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:04.041055+0000 mon.smithi131 (mon.0) 466 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:49:05.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:04.849453+0000 mon.smithi131 (mon.0) 467 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:49:05.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:04.849654+0000 mon.smithi131 (mon.0) 468 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:49:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:05.041361+0000 mon.smithi131 (mon.0) 469 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-02-19T21:49:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:05.167580+0000 mgr.smithi131.cjubsp (mgr.14164) 179 : audit [DBG] from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:49:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:05.168466+0000 mon.smithi131 (mon.0) 470 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:49:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:05.169965+0000 mon.smithi131 (mon.0) 471 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:49:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:05.170661+0000 mon.smithi131 (mon.0) 472 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:05.451005+0000 mgr.smithi131.cjubsp (mgr.14164) 180 : cluster [DBG] pgmap v148: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:05.041361+0000 mon.smithi131 (mon.0) 469 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-02-19T21:49:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:05.167580+0000 mgr.smithi131.cjubsp (mgr.14164) 179 : audit [DBG] from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:49:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:05.168466+0000 mon.smithi131 (mon.0) 470 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:49:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:05.169965+0000 mon.smithi131 (mon.0) 471 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:49:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:05.170661+0000 mon.smithi131 (mon.0) 472 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:05.451005+0000 mgr.smithi131.cjubsp (mgr.14164) 180 : cluster [DBG] pgmap v148: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:08.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:07.451478+0000 mgr.smithi131.cjubsp (mgr.14164) 181 : cluster [DBG] pgmap v149: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:08.835 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:07.451478+0000 mgr.smithi131.cjubsp (mgr.14164) 181 : cluster [DBG] pgmap v149: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:10.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:49:09.393459+0000 mgr.smithi131.cjubsp (mgr.14164) 182 : cephadm [INF] Detected new or changed devices on smithi196 2024-02-19T21:49:10.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:09.398394+0000 mon.smithi131 (mon.0) 473 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:10.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:09.399327+0000 mon.smithi131 (mon.0) 474 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:09.400113+0000 mon.smithi131 (mon.0) 475 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:49:09.400663+0000 mgr.smithi131.cjubsp (mgr.14164) 183 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 7985M 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:09.406460+0000 mon.smithi131 (mon.0) 476 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:09.412431+0000 mon.smithi131 (mon.0) 477 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:09.416029+0000 mon.smithi131 (mon.0) 478 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:09.417047+0000 mon.smithi131 (mon.0) 479 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:09.417626+0000 mon.smithi131 (mon.0) 480 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:09.451864+0000 mgr.smithi131.cjubsp (mgr.14164) 184 : cluster [DBG] pgmap v150: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:10.114526+0000 mon.smithi196 (mon.1) 8 : audit [INF] from='client.? 172.21.15.196:0/2734522677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b1036e24-9f6d-4314-9f07-9f5045458fbc"}]: dispatch 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:10.115229+0000 mon.smithi131 (mon.0) 481 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b1036e24-9f6d-4314-9f07-9f5045458fbc"}]: dispatch 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:10.118873+0000 mon.smithi131 (mon.0) 482 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b1036e24-9f6d-4314-9f07-9f5045458fbc"}]': finished 2024-02-19T21:49:10.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:10.118944+0000 mon.smithi131 (mon.0) 483 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-02-19T21:49:10.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:10.119054+0000 mon.smithi131 (mon.0) 484 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:49:09.393459+0000 mgr.smithi131.cjubsp (mgr.14164) 182 : cephadm [INF] Detected new or changed devices on smithi196 2024-02-19T21:49:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:09.398394+0000 mon.smithi131 (mon.0) 473 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:09.399327+0000 mon.smithi131 (mon.0) 474 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:09.400113+0000 mon.smithi131 (mon.0) 475 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:49:09.400663+0000 mgr.smithi131.cjubsp (mgr.14164) 183 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 7985M 2024-02-19T21:49:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:09.406460+0000 mon.smithi131 (mon.0) 476 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:09.412431+0000 mon.smithi131 (mon.0) 477 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:09.416029+0000 mon.smithi131 (mon.0) 478 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:09.417047+0000 mon.smithi131 (mon.0) 479 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:09.417626+0000 mon.smithi131 (mon.0) 480 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:09.451864+0000 mgr.smithi131.cjubsp (mgr.14164) 184 : cluster [DBG] pgmap v150: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:10.114526+0000 mon.smithi196 (mon.1) 8 : audit [INF] from='client.? 172.21.15.196:0/2734522677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b1036e24-9f6d-4314-9f07-9f5045458fbc"}]: dispatch 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:10.115229+0000 mon.smithi131 (mon.0) 481 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b1036e24-9f6d-4314-9f07-9f5045458fbc"}]: dispatch 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:10.118873+0000 mon.smithi131 (mon.0) 482 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b1036e24-9f6d-4314-9f07-9f5045458fbc"}]': finished 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:10.118944+0000 mon.smithi131 (mon.0) 483 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-02-19T21:49:10.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:10.119054+0000 mon.smithi131 (mon.0) 484 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:11.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:10.523206+0000 mon.smithi196 (mon.1) 9 : audit [DBG] from='client.? 172.21.15.196:0/2588788605' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:49:11.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:10.523206+0000 mon.smithi196 (mon.1) 9 : audit [DBG] from='client.? 172.21.15.196:0/2588788605' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:49:12.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:11.452135+0000 mgr.smithi131.cjubsp (mgr.14164) 185 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:12.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:11.452135+0000 mgr.smithi131.cjubsp (mgr.14164) 185 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:14.820 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:13.452590+0000 mgr.smithi131.cjubsp (mgr.14164) 186 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:14.820 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:13.721905+0000 mon.smithi131 (mon.0) 485 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:14.821 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:13.727134+0000 mon.smithi131 (mon.0) 486 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:15.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:13.452590+0000 mgr.smithi131.cjubsp (mgr.14164) 186 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:15.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:13.721905+0000 mon.smithi131 (mon.0) 485 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:15.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:13.727134+0000 mon.smithi131 (mon.0) 486 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:16.944 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:15.453069+0000 mgr.smithi131.cjubsp (mgr.14164) 187 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:16.944 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:16.247768+0000 mon.smithi131 (mon.0) 487 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-19T21:49:16.944 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:16.248575+0000 mon.smithi131 (mon.0) 488 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:15.453069+0000 mgr.smithi131.cjubsp (mgr.14164) 187 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:16.247768+0000 mon.smithi131 (mon.0) 487 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-19T21:49:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:16.248575+0000 mon.smithi131 (mon.0) 488 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:17.891 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:49:16.249204+0000 mgr.smithi131.cjubsp (mgr.14164) 188 : cephadm [INF] Deploying daemon osd.6 on smithi196 2024-02-19T21:49:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:49:16.249204+0000 mgr.smithi131.cjubsp (mgr.14164) 188 : cephadm [INF] Deploying daemon osd.6 on smithi196 2024-02-19T21:49:18.884 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:17.453565+0000 mgr.smithi131.cjubsp (mgr.14164) 189 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:19.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:17.453565+0000 mgr.smithi131.cjubsp (mgr.14164) 189 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:19.752 INFO:teuthology.orchestra.run.smithi196.stdout:Created osd(s) 6 on host 'smithi196' 2024-02-19T21:49:19.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:19.739281+0000 mon.smithi131 (mon.0) 489 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:19.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:19.741248+0000 mon.smithi131 (mon.0) 490 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:19.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:19.743999+0000 mon.smithi131 (mon.0) 491 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:19.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:19.744255+0000 mon.smithi131 (mon.0) 492 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:20.086 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:19.739281+0000 mon.smithi131 (mon.0) 489 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:20.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:19.741248+0000 mon.smithi131 (mon.0) 490 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:20.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:19.743999+0000 mon.smithi131 (mon.0) 491 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:20.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:19.744255+0000 mon.smithi131 (mon.0) 492 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:20.434 DEBUG:teuthology.orchestra.run.smithi196:osd.6> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.6.service 2024-02-19T21:49:20.436 INFO:tasks.cephadm:Deploying osd.7 on smithi196 with /dev/vg_nvme/lv_1... 2024-02-19T21:49:20.436 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-02-19T21:49:20.467 INFO:journalctl@ceph.osd.6.smithi196.stdout:-- Logs begin at Mon 2024-02-19 21:36:23 UTC. -- 2024-02-19T21:49:21.025 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:19.454076+0000 mgr.smithi131.cjubsp (mgr.14164) 190 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:21.026 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:19.749059+0000 mon.smithi131 (mon.0) 493 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:21.094 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:19.454076+0000 mgr.smithi131.cjubsp (mgr.14164) 190 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:21.095 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:19.749059+0000 mon.smithi131 (mon.0) 493 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:23.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:21.454393+0000 mgr.smithi131.cjubsp (mgr.14164) 191 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:23.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:21.454393+0000 mgr.smithi131.cjubsp (mgr.14164) 191 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:23.174 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:49:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6[46302]: debug 2024-02-19T21:49:22.859+0000 7efc121f5080 -1 osd.6 0 log_to_monitors {default=true} 2024-02-19T21:49:23.201 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-02-19T21:49:23.201 INFO:teuthology.orchestra.run.smithi196.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-02-19T21:49:23.201 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10+0 records in 2024-02-19T21:49:23.201 INFO:teuthology.orchestra.run.smithi196.stderr:10+0 records out 2024-02-19T21:49:23.201 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0143577 s, 730 MB/s 2024-02-19T21:49:23.202 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping successful for: 2024-02-19T21:49:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:22.863743+0000 mon.smithi196 (mon.1) 10 : audit [INF] from='osd.6 [v2:172.21.15.196:6816/3118467872,v1:172.21.15.196:6817/3118467872]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-19T21:49:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:22.864490+0000 mon.smithi131 (mon.0) 494 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-19T21:49:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:22.863743+0000 mon.smithi196 (mon.1) 10 : audit [INF] from='osd.6 [v2:172.21.15.196:6816/3118467872,v1:172.21.15.196:6817/3118467872]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-19T21:49:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:22.864490+0000 mon.smithi131 (mon.0) 494 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-19T21:49:24.328 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch daemon add osd smithi196:vg_nvme/lv_1 2024-02-19T21:49:25.064 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6[46302]: debug 2024-02-19T21:49:24.773+0000 7efc08758700 -1 osd.6 0 waiting for initial osdmap 2024-02-19T21:49:25.064 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6[46302]: debug 2024-02-19T21:49:24.779+0000 7efc036d1700 -1 osd.6 41 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-02-19T21:49:25.064 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:23.454676+0000 mgr.smithi131.cjubsp (mgr.14164) 192 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:25.064 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:23.770958+0000 mon.smithi131 (mon.0) 495 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-19T21:49:25.064 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:23.771015+0000 mon.smithi131 (mon.0) 496 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-02-19T21:49:25.065 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:23.771120+0000 mon.smithi131 (mon.0) 497 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:25.065 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:23.771559+0000 mon.smithi196 (mon.1) 11 : audit [INF] from='osd.6 [v2:172.21.15.196:6816/3118467872,v1:172.21.15.196:6817/3118467872]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:25.065 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:23.771973+0000 mon.smithi131 (mon.0) 498 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:25.065 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:23.864564+0000 mon.smithi131 (mon.0) 499 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:25.065 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:24.461293+0000 mon.smithi131 (mon.0) 500 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:23.454676+0000 mgr.smithi131.cjubsp (mgr.14164) 192 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:23.770958+0000 mon.smithi131 (mon.0) 495 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-19T21:49:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:23.771015+0000 mon.smithi131 (mon.0) 496 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-02-19T21:49:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:23.771120+0000 mon.smithi131 (mon.0) 497 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:23.771559+0000 mon.smithi196 (mon.1) 11 : audit [INF] from='osd.6 [v2:172.21.15.196:6816/3118467872,v1:172.21.15.196:6817/3118467872]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:23.771973+0000 mon.smithi131 (mon.0) 498 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:23.864564+0000 mon.smithi131 (mon.0) 499 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:24.461293+0000 mon.smithi131 (mon.0) 500 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:25.992 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:24.770357+0000 mon.smithi131 (mon.0) 501 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-02-19T21:49:25.992 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:24.770454+0000 mon.smithi131 (mon.0) 502 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-02-19T21:49:25.992 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:24.770929+0000 mon.smithi131 (mon.0) 503 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:25.992 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:24.775106+0000 mon.smithi131 (mon.0) 504 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:24.770357+0000 mon.smithi131 (mon.0) 501 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-02-19T21:49:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:24.770454+0000 mon.smithi131 (mon.0) 502 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-02-19T21:49:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:24.770929+0000 mon.smithi131 (mon.0) 503 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:26.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:24.775106+0000 mon.smithi131 (mon.0) 504 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:27.059 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:23.852977+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:49:27.059 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:23.853155+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:49:27.059 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:25.455128+0000 mgr.smithi131.cjubsp (mgr.14164) 193 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:27.059 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:25.772889+0000 mon.smithi131 (mon.0) 505 : cluster [INF] osd.6 [v2:172.21.15.196:6816/3118467872,v1:172.21.15.196:6817/3118467872] boot 2024-02-19T21:49:27.059 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:25.772979+0000 mon.smithi131 (mon.0) 506 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-02-19T21:49:27.060 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:25.773148+0000 mon.smithi131 (mon.0) 507 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:23.852977+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:49:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:23.853155+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:49:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:25.455128+0000 mgr.smithi131.cjubsp (mgr.14164) 193 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-02-19T21:49:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:25.772889+0000 mon.smithi131 (mon.0) 505 : cluster [INF] osd.6 [v2:172.21.15.196:6816/3118467872,v1:172.21.15.196:6817/3118467872] boot 2024-02-19T21:49:27.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:25.772979+0000 mon.smithi131 (mon.0) 506 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-02-19T21:49:27.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:25.773148+0000 mon.smithi131 (mon.0) 507 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:49:28.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:26.779012+0000 mon.smithi131 (mon.0) 508 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-02-19T21:49:28.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:27.780293+0000 mon.smithi131 (mon.0) 509 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-02-19T21:49:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:26.779012+0000 mon.smithi131 (mon.0) 508 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-02-19T21:49:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:27.780293+0000 mon.smithi131 (mon.0) 509 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:27.455676+0000 mgr.smithi131.cjubsp (mgr.14164) 194 : cluster [DBG] pgmap v164: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:27.916510+0000 mgr.smithi131.cjubsp (mgr.14164) 195 : audit [DBG] from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:27.917401+0000 mon.smithi131 (mon.0) 510 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:27.918968+0000 mon.smithi131 (mon.0) 511 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:27.919638+0000 mon.smithi131 (mon.0) 512 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.470160+0000 mon.smithi131 (mon.0) 513 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.471127+0000 mon.smithi131 (mon.0) 514 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.471862+0000 mon.smithi131 (mon.0) 515 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.472480+0000 mon.smithi131 (mon.0) 516 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:29.125 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.476443+0000 mon.smithi131 (mon.0) 517 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:29.126 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.479962+0000 mon.smithi131 (mon.0) 518 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:29.126 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.483488+0000 mon.smithi131 (mon.0) 519 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:29.126 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.484456+0000 mon.smithi131 (mon.0) 520 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:29.126 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.485057+0000 mon.smithi131 (mon.0) 521 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:29.126 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:28.488903+0000 mon.smithi131 (mon.0) 522 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:27.455676+0000 mgr.smithi131.cjubsp (mgr.14164) 194 : cluster [DBG] pgmap v164: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:27.916510+0000 mgr.smithi131.cjubsp (mgr.14164) 195 : audit [DBG] from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:49:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:27.917401+0000 mon.smithi131 (mon.0) 510 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-19T21:49:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:27.918968+0000 mon.smithi131 (mon.0) 511 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-19T21:49:29.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:27.919638+0000 mon.smithi131 (mon.0) 512 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:29.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.470160+0000 mon.smithi131 (mon.0) 513 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:29.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.471127+0000 mon.smithi131 (mon.0) 514 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:29.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.471862+0000 mon.smithi131 (mon.0) 515 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:29.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.472480+0000 mon.smithi131 (mon.0) 516 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:29.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.476443+0000 mon.smithi131 (mon.0) 517 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:29.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.479962+0000 mon.smithi131 (mon.0) 518 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:29.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.483488+0000 mon.smithi131 (mon.0) 519 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:29.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.484456+0000 mon.smithi131 (mon.0) 520 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:29.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.485057+0000 mon.smithi131 (mon.0) 521 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:29.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:28.488903+0000 mon.smithi131 (mon.0) 522 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:30.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:49:28.464739+0000 mgr.smithi131.cjubsp (mgr.14164) 196 : cephadm [INF] Detected new or changed devices on smithi196 2024-02-19T21:49:30.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:49:28.472958+0000 mgr.smithi131.cjubsp (mgr.14164) 197 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 5323M 2024-02-19T21:49:30.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:49:28.464739+0000 mgr.smithi131.cjubsp (mgr.14164) 196 : cephadm [INF] Detected new or changed devices on smithi196 2024-02-19T21:49:30.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:49:28.472958+0000 mgr.smithi131.cjubsp (mgr.14164) 197 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 5323M 2024-02-19T21:49:31.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:29.456174+0000 mgr.smithi131.cjubsp (mgr.14164) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:31.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:30.455607+0000 mon.smithi196 (mon.1) 12 : audit [INF] from='client.? 172.21.15.196:0/3757600455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d633d620-9d71-4ce8-b94b-d78ee8b5de08"}]: dispatch 2024-02-19T21:49:31.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:30.456218+0000 mon.smithi131 (mon.0) 523 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d633d620-9d71-4ce8-b94b-d78ee8b5de08"}]: dispatch 2024-02-19T21:49:31.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:30.461458+0000 mon.smithi131 (mon.0) 524 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d633d620-9d71-4ce8-b94b-d78ee8b5de08"}]': finished 2024-02-19T21:49:31.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:30.461525+0000 mon.smithi131 (mon.0) 525 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-02-19T21:49:31.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:30.461678+0000 mon.smithi131 (mon.0) 526 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:29.456174+0000 mgr.smithi131.cjubsp (mgr.14164) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:30.455607+0000 mon.smithi196 (mon.1) 12 : audit [INF] from='client.? 172.21.15.196:0/3757600455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d633d620-9d71-4ce8-b94b-d78ee8b5de08"}]: dispatch 2024-02-19T21:49:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:30.456218+0000 mon.smithi131 (mon.0) 523 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d633d620-9d71-4ce8-b94b-d78ee8b5de08"}]: dispatch 2024-02-19T21:49:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:30.461458+0000 mon.smithi131 (mon.0) 524 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d633d620-9d71-4ce8-b94b-d78ee8b5de08"}]': finished 2024-02-19T21:49:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:30.461525+0000 mon.smithi131 (mon.0) 525 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-02-19T21:49:31.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:30.461678+0000 mon.smithi131 (mon.0) 526 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:30.883429+0000 mon.smithi196 (mon.1) 13 : audit [DBG] from='client.? 172.21.15.196:0/1726241862' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:49:32.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:30.883429+0000 mon.smithi196 (mon.1) 13 : audit [DBG] from='client.? 172.21.15.196:0/1726241862' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-19T21:49:33.118 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:31.456685+0000 mgr.smithi131.cjubsp (mgr.14164) 199 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:33.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:31.456685+0000 mgr.smithi131.cjubsp (mgr.14164) 199 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:35.142 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:33.457083+0000 mgr.smithi131.cjubsp (mgr.14164) 200 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:35.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:33.457083+0000 mgr.smithi131.cjubsp (mgr.14164) 200 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:35.457413+0000 mgr.smithi131.cjubsp (mgr.14164) 201 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:36.602212+0000 mon.smithi131 (mon.0) 527 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-19T21:49:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:36.602973+0000 mon.smithi131 (mon.0) 528 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:37.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:35.457413+0000 mgr.smithi131.cjubsp (mgr.14164) 201 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:37.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:36.602212+0000 mon.smithi131 (mon.0) 527 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-19T21:49:37.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:36.602973+0000 mon.smithi131 (mon.0) 528 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:49:36.603550+0000 mgr.smithi131.cjubsp (mgr.14164) 202 : cephadm [INF] Deploying daemon osd.7 on smithi196 2024-02-19T21:49:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:49:36.603550+0000 mgr.smithi131.cjubsp (mgr.14164) 202 : cephadm [INF] Deploying daemon osd.7 on smithi196 2024-02-19T21:49:38.887 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:37.457768+0000 mgr.smithi131.cjubsp (mgr.14164) 203 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:39.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:37.457768+0000 mgr.smithi131.cjubsp (mgr.14164) 203 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:40.474 INFO:teuthology.orchestra.run.smithi196.stdout:Created osd(s) 7 on host 'smithi196' 2024-02-19T21:49:41.132 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:39.458051+0000 mgr.smithi131.cjubsp (mgr.14164) 204 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:41.133 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:40.459487+0000 mon.smithi131 (mon.0) 529 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:41.133 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:40.461584+0000 mon.smithi131 (mon.0) 530 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:41.133 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:40.462325+0000 mon.smithi131 (mon.0) 531 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:41.133 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:40.465967+0000 mon.smithi131 (mon.0) 532 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:41.133 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:40.472871+0000 mon.smithi131 (mon.0) 533 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:41.156 DEBUG:teuthology.orchestra.run.smithi196:osd.7> sudo journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.7.service 2024-02-19T21:49:41.158 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-02-19T21:49:41.159 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd stat -f json 2024-02-19T21:49:41.188 INFO:journalctl@ceph.osd.7.smithi196.stdout:-- Logs begin at Mon 2024-02-19 21:36:23 UTC. -- 2024-02-19T21:49:41.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:39.458051+0000 mgr.smithi131.cjubsp (mgr.14164) 204 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:41.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:40.459487+0000 mon.smithi131 (mon.0) 529 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:41.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:40.461584+0000 mon.smithi131 (mon.0) 530 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:41.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:40.462325+0000 mon.smithi131 (mon.0) 531 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:41.197 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:40.465967+0000 mon.smithi131 (mon.0) 532 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:41.197 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:40.472871+0000 mon.smithi131 (mon.0) 533 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:42.954 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:41.458359+0000 mgr.smithi131.cjubsp (mgr.14164) 205 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:43.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:41.458359+0000 mgr.smithi131.cjubsp (mgr.14164) 205 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:43.347 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:49:43.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:49:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.7[49192]: debug 2024-02-19T21:49:43.580+0000 7f9e2302d080 -1 osd.7 0 log_to_monitors {default=true} 2024-02-19T21:49:43.938 INFO:teuthology.orchestra.run.smithi131.stdout:{"epoch":45,"num_osds":8,"num_up_osds":7,"osd_up_since":1708379365,"num_in_osds":8,"osd_in_since":1708379370,"num_remapped_pgs":0} 2024-02-19T21:49:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:43.166661+0000 mon.smithi131 (mon.0) 534 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:43.348047+0000 mon.smithi131 (mon.0) 535 : audit [DBG] from='client.? 172.21.15.131:0/1100268764' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-19T21:49:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:43.458674+0000 mgr.smithi131.cjubsp (mgr.14164) 206 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:43.584015+0000 mon.smithi196 (mon.1) 14 : audit [INF] from='osd.7 [v2:172.21.15.196:6824/1112062173,v1:172.21.15.196:6825/1112062173]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-19T21:49:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:43.584473+0000 mon.smithi131 (mon.0) 536 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-19T21:49:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:43.166661+0000 mon.smithi131 (mon.0) 534 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:43.348047+0000 mon.smithi131 (mon.0) 535 : audit [DBG] from='client.? 172.21.15.131:0/1100268764' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-19T21:49:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:43.458674+0000 mgr.smithi131.cjubsp (mgr.14164) 206 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:43.584015+0000 mon.smithi196 (mon.1) 14 : audit [INF] from='osd.7 [v2:172.21.15.196:6824/1112062173,v1:172.21.15.196:6825/1112062173]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-19T21:49:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:43.584473+0000 mon.smithi131 (mon.0) 536 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-19T21:49:44.940 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd stat -f json 2024-02-19T21:49:45.442 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:44.174401+0000 mon.smithi131 (mon.0) 537 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-19T21:49:45.442 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:44.174464+0000 mon.smithi131 (mon.0) 538 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-02-19T21:49:45.443 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:44.174778+0000 mon.smithi131 (mon.0) 539 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:45.443 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:44.174911+0000 mon.smithi196 (mon.1) 15 : audit [INF] from='osd.7 [v2:172.21.15.196:6824/1112062173,v1:172.21.15.196:6825/1112062173]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:45.443 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:44.175274+0000 mon.smithi131 (mon.0) 540 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:45.654 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:49:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.7[49192]: debug 2024-02-19T21:49:45.180+0000 7f9e19590700 -1 osd.7 0 waiting for initial osdmap 2024-02-19T21:49:45.654 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:49:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.7[49192]: debug 2024-02-19T21:49:45.188+0000 7f9e13507700 -1 osd.7 47 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-02-19T21:49:45.654 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:44.174401+0000 mon.smithi131 (mon.0) 537 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-19T21:49:45.654 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:44.174464+0000 mon.smithi131 (mon.0) 538 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-02-19T21:49:45.654 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:44.174778+0000 mon.smithi131 (mon.0) 539 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:45.654 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:44.174911+0000 mon.smithi196 (mon.1) 15 : audit [INF] from='osd.7 [v2:172.21.15.196:6824/1112062173,v1:172.21.15.196:6825/1112062173]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:45.655 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:44.175274+0000 mon.smithi131 (mon.0) 540 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:49:46.346 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:45.175346+0000 mon.smithi131 (mon.0) 541 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-02-19T21:49:46.346 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:45.175438+0000 mon.smithi131 (mon.0) 542 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-02-19T21:49:46.346 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:45.175802+0000 mon.smithi131 (mon.0) 543 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:46.346 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:45.180130+0000 mon.smithi131 (mon.0) 544 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:46.346 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:45.459019+0000 mgr.smithi131.cjubsp (mgr.14164) 207 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:46.461 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:45.175346+0000 mon.smithi131 (mon.0) 541 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-02-19T21:49:46.461 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:45.175438+0000 mon.smithi131 (mon.0) 542 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-02-19T21:49:46.461 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:45.175802+0000 mon.smithi131 (mon.0) 543 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:46.462 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:45.180130+0000 mon.smithi131 (mon.0) 544 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:46.462 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:45.459019+0000 mgr.smithi131.cjubsp (mgr.14164) 207 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-02-19T21:49:47.118 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:44.542272+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:44.542423+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:46.177428+0000 mon.smithi131 (mon.0) 545 : cluster [INF] osd.7 [v2:172.21.15.196:6824/1112062173,v1:172.21.15.196:6825/1112062173] boot 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:46.177472+0000 mon.smithi131 (mon.0) 546 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.177592+0000 mon.smithi131 (mon.0) 547 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:49:46.569404+0000 mgr.smithi131.cjubsp (mgr.14164) 208 : cephadm [INF] Detected new or changed devices on smithi196 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.573947+0000 mon.smithi131 (mon.0) 548 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.574781+0000 mon.smithi131 (mon.0) 549 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.575457+0000 mon.smithi131 (mon.0) 550 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.576058+0000 mon.smithi131 (mon.0) 551 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:47.463 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.576713+0000 mon.smithi131 (mon.0) 552 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:49:46.577135+0000 mgr.smithi131.cjubsp (mgr.14164) 209 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 3992M 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.580528+0000 mon.smithi131 (mon.0) 553 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.584272+0000 mon.smithi131 (mon.0) 554 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.587374+0000 mon.smithi131 (mon.0) 555 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.588237+0000 mon.smithi131 (mon.0) 556 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.588795+0000 mon.smithi131 (mon.0) 557 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:46.592269+0000 mon.smithi131 (mon.0) 558 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:47.118800+0000 mon.smithi131 (mon.0) 559 : audit [DBG] from='client.? 172.21.15.131:0/3694258395' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-19T21:49:47.464 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:47.177672+0000 mon.smithi131 (mon.0) 560 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-02-19T21:49:47.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:44.542272+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:44.542423+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:46.177428+0000 mon.smithi131 (mon.0) 545 : cluster [INF] osd.7 [v2:172.21.15.196:6824/1112062173,v1:172.21.15.196:6825/1112062173] boot 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:46.177472+0000 mon.smithi131 (mon.0) 546 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.177592+0000 mon.smithi131 (mon.0) 547 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:49:46.569404+0000 mgr.smithi131.cjubsp (mgr.14164) 208 : cephadm [INF] Detected new or changed devices on smithi196 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.573947+0000 mon.smithi131 (mon.0) 548 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.574781+0000 mon.smithi131 (mon.0) 549 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.575457+0000 mon.smithi131 (mon.0) 550 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.576058+0000 mon.smithi131 (mon.0) 551 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.576713+0000 mon.smithi131 (mon.0) 552 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:49:46.577135+0000 mgr.smithi131.cjubsp (mgr.14164) 209 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 3992M 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.580528+0000 mon.smithi131 (mon.0) 553 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:47.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.584272+0000 mon.smithi131 (mon.0) 554 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:47.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.587374+0000 mon.smithi131 (mon.0) 555 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:49:47.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.588237+0000 mon.smithi131 (mon.0) 556 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:49:47.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.588795+0000 mon.smithi131 (mon.0) 557 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:49:47.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:46.592269+0000 mon.smithi131 (mon.0) 558 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:49:47.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:47.118800+0000 mon.smithi131 (mon.0) 559 : audit [DBG] from='client.? 172.21.15.131:0/3694258395' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-19T21:49:47.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:47.177672+0000 mon.smithi131 (mon.0) 560 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-02-19T21:49:47.770 INFO:teuthology.orchestra.run.smithi131.stdout:{"epoch":48,"num_osds":8,"num_up_osds":8,"osd_up_since":1708379386,"num_in_osds":8,"osd_in_since":1708379370,"num_remapped_pgs":1} 2024-02-19T21:49:47.771 INFO:tasks.cephadm:Setting up client nodes... 2024-02-19T21:49:47.771 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-02-19T21:49:48.461 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:47.459376+0000 mgr.smithi131.cjubsp (mgr.14164) 210 : cluster [DBG] pgmap v180: 1 pgs: 1 remapped+peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:48.461 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:48.179765+0000 mon.smithi131 (mon.0) 561 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-02-19T21:49:48.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:47.459376+0000 mgr.smithi131.cjubsp (mgr.14164) 210 : cluster [DBG] pgmap v180: 1 pgs: 1 remapped+peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:48.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:48.179765+0000 mon.smithi131 (mon.0) 561 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-02-19T21:49:49.405 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:48.183308+0000 mon.smithi131 (mon.0) 562 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-02-19T21:49:49.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:48.183308+0000 mon.smithi131 (mon.0) 562 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-02-19T21:49:49.887 INFO:teuthology.orchestra.run.smithi131.stdout:[client.0] 2024-02-19T21:49:49.887 INFO:teuthology.orchestra.run.smithi131.stdout: key = AQD9zNNl4ym/NBAAcU1B5rUqOnRqxwbnm4hxPQ== 2024-02-19T21:49:50.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:49.459680+0000 mgr.smithi131.cjubsp (mgr.14164) 211 : cluster [DBG] pgmap v182: 1 pgs: 1 remapped+peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:50.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:49.884780+0000 mon.smithi131 (mon.0) 563 : audit [INF] from='client.? 172.21.15.131:0/1425711060' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-19T21:49:50.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:49.888020+0000 mon.smithi131 (mon.0) 564 : audit [INF] from='client.? 172.21.15.131:0/1425711060' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-19T21:49:50.659 DEBUG:teuthology.orchestra.run.smithi131:> set -ex 2024-02-19T21:49:50.659 DEBUG:teuthology.orchestra.run.smithi131:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-02-19T21:49:50.660 DEBUG:teuthology.orchestra.run.smithi131:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-02-19T21:49:50.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:49.459680+0000 mgr.smithi131.cjubsp (mgr.14164) 211 : cluster [DBG] pgmap v182: 1 pgs: 1 remapped+peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:50.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:49.884780+0000 mon.smithi131 (mon.0) 563 : audit [INF] from='client.? 172.21.15.131:0/1425711060' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-19T21:49:50.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:49.888020+0000 mon.smithi131 (mon.0) 564 : audit [INF] from='client.? 172.21.15.131:0/1425711060' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-19T21:49:50.701 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd dump --format=json 2024-02-19T21:49:50.962 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:49:51.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:51.510813+0000 mon.smithi131 (mon.0) 565 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-02-19T21:49:51.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:51.510865+0000 mon.smithi131 (mon.0) 566 : cluster [INF] Cluster is now healthy 2024-02-19T21:49:51.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:51.510813+0000 mon.smithi131 (mon.0) 565 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-02-19T21:49:51.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:51.510865+0000 mon.smithi131 (mon.0) 566 : cluster [INF] Cluster is now healthy 2024-02-19T21:49:52.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:51.460147+0000 mgr.smithi131.cjubsp (mgr.14164) 212 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:52.907 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:49:52.907 INFO:teuthology.orchestra.run.smithi131.stdout:{"epoch":50,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","created":"2024-02-19T21:43:58.984120+0000","modified":"2024-02-19T21:49:48.175644+0000","last_up_change":"2024-02-19T21:49:46.172953+0000","last_in_change":"2024-02-19T21:49:30.456860+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-02-19T21:48:03.114027+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"9332188a-8d84-40a8-b382-1eca712a6a3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6802","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6803","nonce":775149687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6804","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6805","nonce":775149687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6808","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6809","nonce":775149687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6806","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6807","nonce":775149687}]},"public_addr":"172.21.15.131:6803/775149687","cluster_addr":"172.21.15.131:6805/775149687","heartbeat_back_addr":"172.21.15.131:6809/775149687","heartbeat_front_addr":"172.21.15.131:6807/775149687","state":["exists","up"]},{"osd":1,"uuid":"bcb5683e-6881-4a06-a14d-0dd0eca432b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6810","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6811","nonce":3016860838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6812","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6813","nonce":3016860838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6816","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6817","nonce":3016860838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6814","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6815","nonce":3016860838}]},"public_addr":"172.21.15.131:6811/3016860838","cluster_addr":"172.21.15.131:6813/3016860838","heartbeat_back_addr":"172.21.15.131:6817/3016860838","heartbeat_front_addr":"172.21.15.131:6815/3016860838","state":["exists","up"]},{"osd":2,"uuid":"52e5f8fc-2bac-4ff7-9cef-fea8652db649","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6818","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6819","nonce":3047587875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6820","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6821","nonce":3047587875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6824","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6825","nonce":3047587875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6822","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6823","nonce":3047587875}]},"public_addr":"172.21.15.131:6819/3047587875","cluster_addr":"172.21.15.131:6821/3047587875","heartbeat_back_addr":"172.21.15.131:6825/3047587875","heartbeat_front_addr":"172.21.15.131:6823/3047587875","state":["exists","up"]},{"osd":3,"uuid":"77028a52-76a4-49b5-9f06-58a428714aa7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6826","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6827","nonce":768749547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6828","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6829","nonce":768749547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6832","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6833","nonce":768749547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6830","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6831","nonce":768749547}]},"public_addr":"172.21.15.131:6827/768749547","cluster_addr":"172.21.15.131:6829/768749547","heartbeat_back_addr":"172.21.15.131:6833/768749547","heartbeat_front_addr":"172.21.15.131:6831/768749547","state":["exists","up"]},{"osd":4,"uuid":"288d638c-1abd-4b32-b4f1-339435dcf230","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6801","nonce":10664072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6803","nonce":10664072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6807","nonce":10664072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6805","nonce":10664072}]},"public_addr":"172.21.15.196:6801/10664072","cluster_addr":"172.21.15.196:6803/10664072","heartbeat_back_addr":"172.21.15.196:6807/10664072","heartbeat_front_addr":"172.21.15.196:6805/10664072","state":["exists","up"]},{"osd":5,"uuid":"734e875d-f0f1-4204-9063-1404485397be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6809","nonce":3462679939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6811","nonce":3462679939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6815","nonce":3462679939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6813","nonce":3462679939}]},"public_addr":"172.21.15.196:6809/3462679939","cluster_addr":"172.21.15.196:6811/3462679939","heartbeat_back_addr":"172.21.15.196:6815/3462679939","heartbeat_front_addr":"172.21.15.196:6813/3462679939","state":["exists","up"]},{"osd":6,"uuid":"b1036e24-9f6d-4314-9f07-9f5045458fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6817","nonce":3118467872}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6819","nonce":3118467872}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6823","nonce":3118467872}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6821","nonce":3118467872}]},"public_addr":"172.21.15.196:6817/3118467872","cluster_addr":"172.21.15.196:6819/3118467872","heartbeat_back_addr":"172.21.15.196:6823/3118467872","heartbeat_front_addr":"172.21.15.196:6821/3118467872","state":["exists","up"]},{"osd":7,"uuid":"d633d620-9d71-4ce8-b94b-d78ee8b5de08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6825","nonce":1112062173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6827","nonce":1112062173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6831","nonce":1112062173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6829","nonce":1112062173}]},"public_addr":"172.21.15.196:6825/1112062173","cluster_addr":"172.21.15.196:6827/1112062173","heartbeat_back_addr":"172.21.15.196:6831/1112062173","heartbeat_front_addr":"172.21.15.196:6829/1112062173","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:47:14.886490+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:47:39.275412+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:01.408750+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:24.401704+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:42.396249+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:02.898744+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:23.853160+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:44.542425+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.131:0/247383117":"2024-02-20T21:45:04.116961+0000","172.21.15.131:6801/2909791757":"2024-02-20T21:45:04.116961+0000","172.21.15.131:6800/2909791757":"2024-02-20T21:45:04.116961+0000","172.21.15.131:0/2715933173":"2024-02-20T21:44:47.985866+0000","172.21.15.131:0/1914610642":"2024-02-20T21:45:04.116961+0000","172.21.15.131:0/1744032712":"2024-02-20T21:44:47.985866+0000","172.21.15.131:6801/2834889514":"2024-02-20T21:44:47.985866+0000","172.21.15.131:6800/2834889514":"2024-02-20T21:44:47.985866+0000","172.21.15.131:0/3636304960":"2024-02-20T21:44:27.578096+0000","172.21.15.131:0/1139724619":"2024-02-20T21:44:27.578096+0000","172.21.15.131:6800/3671695538":"2024-02-20T21:44:27.578096+0000","172.21.15.131:6801/3671695538":"2024-02-20T21:44:27.578096+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-19T21:49:52.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:51.460147+0000 mgr.smithi131.cjubsp (mgr.14164) 212 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:53.658 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2024-02-19T21:48:03.114027+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2024-02-19T21:49:53.658 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd pool get device_health_metrics pg_num 2024-02-19T21:49:53.885 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:49:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:52.906995+0000 mon.smithi131 (mon.0) 567 : audit [DBG] from='client.? 172.21.15.131:0/3371380502' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-19T21:49:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:52.906995+0000 mon.smithi131 (mon.0) 567 : audit [DBG] from='client.? 172.21.15.131:0/3371380502' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-19T21:49:55.146 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:53.460589+0000 mgr.smithi131.cjubsp (mgr.14164) 213 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:53.460589+0000 mgr.smithi131.cjubsp (mgr.14164) 213 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:55.821 INFO:teuthology.orchestra.run.smithi131.stdout:pg_num: 1 2024-02-19T21:49:56.615 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-02-19T21:49:56.615 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-02-19T21:49:56.615 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph mgr dump --format=json 2024-02-19T21:49:56.839 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:49:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:55.460937+0000 mgr.smithi131.cjubsp (mgr.14164) 214 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:55.821893+0000 mon.smithi131 (mon.0) 568 : audit [DBG] from='client.? 172.21.15.131:0/3690304993' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2024-02-19T21:49:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:55.460937+0000 mgr.smithi131.cjubsp (mgr.14164) 214 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:55.821893+0000 mon.smithi131 (mon.0) 568 : audit [DBG] from='client.? 172.21.15.131:0/3690304993' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2024-02-19T21:49:58.954 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:49:59.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:57.461260+0000 mgr.smithi131.cjubsp (mgr.14164) 215 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:59.220 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:57.461260+0000 mgr.smithi131.cjubsp (mgr.14164) 215 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:49:59.648 INFO:teuthology.orchestra.run.smithi131.stdout:{"epoch":18,"active_gid":14164,"active_name":"smithi131.cjubsp","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6800","nonce":2570377912},{"type":"v1","addr":"172.21.15.131:6801","nonce":2570377912}]},"active_addr":"172.21.15.131:6801/2570377912","active_change":"2024-02-19T21:45:04.117274+0000","active_mgr_features":4540138297136906239,"available":true,"standbys":[{"gid":14206,"name":"smithi196.voprex","mgr_features":4540138297136906239,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"docker.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"docker.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Registry to which we should normalize unqualified image names","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"docker.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"docker.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Registry to which we should normalize unqualified image names","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.131:8443/","prometheus":"http://172.21.15.131:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":4,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.131:0","nonce":1629220184}]},{"addrvec":[{"type":"v2","addr":"172.21.15.131:0","nonce":2536315811}]}]}} 2024-02-19T21:49:59.651 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-02-19T21:49:59.651 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-02-19T21:49:59.651 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd dump --format=json 2024-02-19T21:49:59.881 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:00.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:49:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:49:58.950565+0000 mon.smithi131 (mon.0) 569 : audit [DBG] from='client.? 172.21.15.131:0/2834436544' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-19T21:50:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:49:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:49:58.950565+0000 mon.smithi131 (mon.0) 569 : audit [DBG] from='client.? 172.21.15.131:0/2834436544' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-19T21:50:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:49:59.461755+0000 mgr.smithi131.cjubsp (mgr.14164) 216 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:00.000161+0000 mon.smithi131 (mon.0) 570 : cluster [INF] overall HEALTH_OK 2024-02-19T21:50:01.242 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:49:59.461755+0000 mgr.smithi131.cjubsp (mgr.14164) 216 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:01.242 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:00.000161+0000 mon.smithi131 (mon.0) 570 : cluster [INF] overall HEALTH_OK 2024-02-19T21:50:01.920 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:50:01.921 INFO:teuthology.orchestra.run.smithi131.stdout:{"epoch":50,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","created":"2024-02-19T21:43:58.984120+0000","modified":"2024-02-19T21:49:48.175644+0000","last_up_change":"2024-02-19T21:49:46.172953+0000","last_in_change":"2024-02-19T21:49:30.456860+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-02-19T21:48:03.114027+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"9332188a-8d84-40a8-b382-1eca712a6a3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6802","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6803","nonce":775149687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6804","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6805","nonce":775149687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6808","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6809","nonce":775149687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6806","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6807","nonce":775149687}]},"public_addr":"172.21.15.131:6803/775149687","cluster_addr":"172.21.15.131:6805/775149687","heartbeat_back_addr":"172.21.15.131:6809/775149687","heartbeat_front_addr":"172.21.15.131:6807/775149687","state":["exists","up"]},{"osd":1,"uuid":"bcb5683e-6881-4a06-a14d-0dd0eca432b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6810","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6811","nonce":3016860838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6812","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6813","nonce":3016860838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6816","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6817","nonce":3016860838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6814","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6815","nonce":3016860838}]},"public_addr":"172.21.15.131:6811/3016860838","cluster_addr":"172.21.15.131:6813/3016860838","heartbeat_back_addr":"172.21.15.131:6817/3016860838","heartbeat_front_addr":"172.21.15.131:6815/3016860838","state":["exists","up"]},{"osd":2,"uuid":"52e5f8fc-2bac-4ff7-9cef-fea8652db649","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6818","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6819","nonce":3047587875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6820","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6821","nonce":3047587875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6824","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6825","nonce":3047587875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6822","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6823","nonce":3047587875}]},"public_addr":"172.21.15.131:6819/3047587875","cluster_addr":"172.21.15.131:6821/3047587875","heartbeat_back_addr":"172.21.15.131:6825/3047587875","heartbeat_front_addr":"172.21.15.131:6823/3047587875","state":["exists","up"]},{"osd":3,"uuid":"77028a52-76a4-49b5-9f06-58a428714aa7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6826","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6827","nonce":768749547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6828","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6829","nonce":768749547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6832","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6833","nonce":768749547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6830","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6831","nonce":768749547}]},"public_addr":"172.21.15.131:6827/768749547","cluster_addr":"172.21.15.131:6829/768749547","heartbeat_back_addr":"172.21.15.131:6833/768749547","heartbeat_front_addr":"172.21.15.131:6831/768749547","state":["exists","up"]},{"osd":4,"uuid":"288d638c-1abd-4b32-b4f1-339435dcf230","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6801","nonce":10664072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6803","nonce":10664072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6807","nonce":10664072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6805","nonce":10664072}]},"public_addr":"172.21.15.196:6801/10664072","cluster_addr":"172.21.15.196:6803/10664072","heartbeat_back_addr":"172.21.15.196:6807/10664072","heartbeat_front_addr":"172.21.15.196:6805/10664072","state":["exists","up"]},{"osd":5,"uuid":"734e875d-f0f1-4204-9063-1404485397be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6809","nonce":3462679939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6811","nonce":3462679939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6815","nonce":3462679939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6813","nonce":3462679939}]},"public_addr":"172.21.15.196:6809/3462679939","cluster_addr":"172.21.15.196:6811/3462679939","heartbeat_back_addr":"172.21.15.196:6815/3462679939","heartbeat_front_addr":"172.21.15.196:6813/3462679939","state":["exists","up"]},{"osd":6,"uuid":"b1036e24-9f6d-4314-9f07-9f5045458fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6817","nonce":3118467872}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6819","nonce":3118467872}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6823","nonce":3118467872}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6821","nonce":3118467872}]},"public_addr":"172.21.15.196:6817/3118467872","cluster_addr":"172.21.15.196:6819/3118467872","heartbeat_back_addr":"172.21.15.196:6823/3118467872","heartbeat_front_addr":"172.21.15.196:6821/3118467872","state":["exists","up"]},{"osd":7,"uuid":"d633d620-9d71-4ce8-b94b-d78ee8b5de08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6825","nonce":1112062173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6827","nonce":1112062173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6831","nonce":1112062173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6829","nonce":1112062173}]},"public_addr":"172.21.15.196:6825/1112062173","cluster_addr":"172.21.15.196:6827/1112062173","heartbeat_back_addr":"172.21.15.196:6831/1112062173","heartbeat_front_addr":"172.21.15.196:6829/1112062173","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:47:14.886490+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:47:39.275412+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:01.408750+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:24.401704+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:42.396249+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:02.898744+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:23.853160+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:44.542425+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.131:0/247383117":"2024-02-20T21:45:04.116961+0000","172.21.15.131:6801/2909791757":"2024-02-20T21:45:04.116961+0000","172.21.15.131:6800/2909791757":"2024-02-20T21:45:04.116961+0000","172.21.15.131:0/2715933173":"2024-02-20T21:44:47.985866+0000","172.21.15.131:0/1914610642":"2024-02-20T21:45:04.116961+0000","172.21.15.131:0/1744032712":"2024-02-20T21:44:47.985866+0000","172.21.15.131:6801/2834889514":"2024-02-20T21:44:47.985866+0000","172.21.15.131:6800/2834889514":"2024-02-20T21:44:47.985866+0000","172.21.15.131:0/3636304960":"2024-02-20T21:44:27.578096+0000","172.21.15.131:0/1139724619":"2024-02-20T21:44:27.578096+0000","172.21.15.131:6800/3671695538":"2024-02-20T21:44:27.578096+0000","172.21.15.131:6801/3671695538":"2024-02-20T21:44:27.578096+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-19T21:50:02.527 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-02-19T21:50:02.528 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd dump --format=json 2024-02-19T21:50:02.752 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:01.462268+0000 mgr.smithi131.cjubsp (mgr.14164) 217 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:01.920876+0000 mon.smithi131 (mon.0) 571 : audit [DBG] from='client.? 172.21.15.131:0/3818236471' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-19T21:50:03.191 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:01.462268+0000 mgr.smithi131.cjubsp (mgr.14164) 217 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:03.191 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:01.920876+0000 mon.smithi131 (mon.0) 571 : audit [DBG] from='client.? 172.21.15.131:0/3818236471' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-19T21:50:04.509 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:50:04.509 INFO:teuthology.orchestra.run.smithi131.stdout:{"epoch":50,"fsid":"e8e4a71a-cf6f-11ee-95bb-87774f69a715","created":"2024-02-19T21:43:58.984120+0000","modified":"2024-02-19T21:49:48.175644+0000","last_up_change":"2024-02-19T21:49:46.172953+0000","last_in_change":"2024-02-19T21:49:30.456860+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-02-19T21:48:03.114027+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"9332188a-8d84-40a8-b382-1eca712a6a3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6802","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6803","nonce":775149687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6804","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6805","nonce":775149687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6808","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6809","nonce":775149687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6806","nonce":775149687},{"type":"v1","addr":"172.21.15.131:6807","nonce":775149687}]},"public_addr":"172.21.15.131:6803/775149687","cluster_addr":"172.21.15.131:6805/775149687","heartbeat_back_addr":"172.21.15.131:6809/775149687","heartbeat_front_addr":"172.21.15.131:6807/775149687","state":["exists","up"]},{"osd":1,"uuid":"bcb5683e-6881-4a06-a14d-0dd0eca432b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6810","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6811","nonce":3016860838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6812","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6813","nonce":3016860838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6816","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6817","nonce":3016860838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6814","nonce":3016860838},{"type":"v1","addr":"172.21.15.131:6815","nonce":3016860838}]},"public_addr":"172.21.15.131:6811/3016860838","cluster_addr":"172.21.15.131:6813/3016860838","heartbeat_back_addr":"172.21.15.131:6817/3016860838","heartbeat_front_addr":"172.21.15.131:6815/3016860838","state":["exists","up"]},{"osd":2,"uuid":"52e5f8fc-2bac-4ff7-9cef-fea8652db649","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6818","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6819","nonce":3047587875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6820","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6821","nonce":3047587875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6824","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6825","nonce":3047587875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6822","nonce":3047587875},{"type":"v1","addr":"172.21.15.131:6823","nonce":3047587875}]},"public_addr":"172.21.15.131:6819/3047587875","cluster_addr":"172.21.15.131:6821/3047587875","heartbeat_back_addr":"172.21.15.131:6825/3047587875","heartbeat_front_addr":"172.21.15.131:6823/3047587875","state":["exists","up"]},{"osd":3,"uuid":"77028a52-76a4-49b5-9f06-58a428714aa7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6826","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6827","nonce":768749547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6828","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6829","nonce":768749547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6832","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6833","nonce":768749547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.131:6830","nonce":768749547},{"type":"v1","addr":"172.21.15.131:6831","nonce":768749547}]},"public_addr":"172.21.15.131:6827/768749547","cluster_addr":"172.21.15.131:6829/768749547","heartbeat_back_addr":"172.21.15.131:6833/768749547","heartbeat_front_addr":"172.21.15.131:6831/768749547","state":["exists","up"]},{"osd":4,"uuid":"288d638c-1abd-4b32-b4f1-339435dcf230","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6801","nonce":10664072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6803","nonce":10664072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6807","nonce":10664072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":10664072},{"type":"v1","addr":"172.21.15.196:6805","nonce":10664072}]},"public_addr":"172.21.15.196:6801/10664072","cluster_addr":"172.21.15.196:6803/10664072","heartbeat_back_addr":"172.21.15.196:6807/10664072","heartbeat_front_addr":"172.21.15.196:6805/10664072","state":["exists","up"]},{"osd":5,"uuid":"734e875d-f0f1-4204-9063-1404485397be","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6809","nonce":3462679939}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6811","nonce":3462679939}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6815","nonce":3462679939}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":3462679939},{"type":"v1","addr":"172.21.15.196:6813","nonce":3462679939}]},"public_addr":"172.21.15.196:6809/3462679939","cluster_addr":"172.21.15.196:6811/3462679939","heartbeat_back_addr":"172.21.15.196:6815/3462679939","heartbeat_front_addr":"172.21.15.196:6813/3462679939","state":["exists","up"]},{"osd":6,"uuid":"b1036e24-9f6d-4314-9f07-9f5045458fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6817","nonce":3118467872}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6819","nonce":3118467872}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6823","nonce":3118467872}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":3118467872},{"type":"v1","addr":"172.21.15.196:6821","nonce":3118467872}]},"public_addr":"172.21.15.196:6817/3118467872","cluster_addr":"172.21.15.196:6819/3118467872","heartbeat_back_addr":"172.21.15.196:6823/3118467872","heartbeat_front_addr":"172.21.15.196:6821/3118467872","state":["exists","up"]},{"osd":7,"uuid":"d633d620-9d71-4ce8-b94b-d78ee8b5de08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6825","nonce":1112062173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6827","nonce":1112062173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6831","nonce":1112062173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":1112062173},{"type":"v1","addr":"172.21.15.196:6829","nonce":1112062173}]},"public_addr":"172.21.15.196:6825/1112062173","cluster_addr":"172.21.15.196:6827/1112062173","heartbeat_back_addr":"172.21.15.196:6831/1112062173","heartbeat_front_addr":"172.21.15.196:6829/1112062173","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:47:14.886490+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:47:39.275412+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:01.408750+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:24.401704+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:48:42.396249+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:02.898744+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:23.853160+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-02-19T21:49:44.542425+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.131:0/247383117":"2024-02-20T21:45:04.116961+0000","172.21.15.131:6801/2909791757":"2024-02-20T21:45:04.116961+0000","172.21.15.131:6800/2909791757":"2024-02-20T21:45:04.116961+0000","172.21.15.131:0/2715933173":"2024-02-20T21:44:47.985866+0000","172.21.15.131:0/1914610642":"2024-02-20T21:45:04.116961+0000","172.21.15.131:0/1744032712":"2024-02-20T21:44:47.985866+0000","172.21.15.131:6801/2834889514":"2024-02-20T21:44:47.985866+0000","172.21.15.131:6800/2834889514":"2024-02-20T21:44:47.985866+0000","172.21.15.131:0/3636304960":"2024-02-20T21:44:27.578096+0000","172.21.15.131:0/1139724619":"2024-02-20T21:44:27.578096+0000","172.21.15.131:6800/3671695538":"2024-02-20T21:44:27.578096+0000","172.21.15.131:6801/3671695538":"2024-02-20T21:44:27.578096+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-19T21:50:04.915 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:03.462626+0000 mgr.smithi131.cjubsp (mgr.14164) 218 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:04.916 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:04.509779+0000 mon.smithi131 (mon.0) 572 : audit [DBG] from='client.? 172.21.15.131:0/2960487845' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-19T21:50:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:03.462626+0000 mgr.smithi131.cjubsp (mgr.14164) 218 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:04.509779+0000 mon.smithi131 (mon.0) 572 : audit [DBG] from='client.? 172.21.15.131:0/2960487845' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-19T21:50:05.327 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-02-19T21:50:05.552 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:04.851891+0000 mon.smithi131 (mon.0) 573 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:50:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:04.852090+0000 mon.smithi131 (mon.0) 574 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:50:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:04.851891+0000 mon.smithi131 (mon.0) 573 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:50:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:04.852090+0000 mon.smithi131 (mon.0) 574 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:50:06.952 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:05.462960+0000 mgr.smithi131.cjubsp (mgr.14164) 219 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:05.462960+0000 mgr.smithi131.cjubsp (mgr.14164) 219 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:08.006 INFO:teuthology.orchestra.run.smithi131.stdout:38654705701 2024-02-19T21:50:08.007 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-02-19T21:50:08.232 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:09.104 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:07.463315+0000 mgr.smithi131.cjubsp (mgr.14164) 220 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:07.463315+0000 mgr.smithi131.cjubsp (mgr.14164) 220 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:10.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:09.463657+0000 mgr.smithi131.cjubsp (mgr.14164) 221 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:10.964 INFO:teuthology.orchestra.run.smithi131.stdout:60129542176 2024-02-19T21:50:10.964 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-02-19T21:50:11.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:09.463657+0000 mgr.smithi131.cjubsp (mgr.14164) 221 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:11.189 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:13.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:11.464139+0000 mgr.smithi131.cjubsp (mgr.14164) 222 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:13.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:11.464139+0000 mgr.smithi131.cjubsp (mgr.14164) 222 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:13.661 INFO:teuthology.orchestra.run.smithi131.stdout:81604378653 2024-02-19T21:50:13.661 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-02-19T21:50:13.890 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:15.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:13.464762+0000 mgr.smithi131.cjubsp (mgr.14164) 223 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:13.464762+0000 mgr.smithi131.cjubsp (mgr.14164) 223 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:16.418 INFO:teuthology.orchestra.run.smithi131.stdout:111669149720 2024-02-19T21:50:16.418 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-02-19T21:50:16.645 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:17.096 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:15.465172+0000 mgr.smithi131.cjubsp (mgr.14164) 224 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:17.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:15.465172+0000 mgr.smithi131.cjubsp (mgr.14164) 224 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:19.111 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:17.465519+0000 mgr.smithi131.cjubsp (mgr.14164) 225 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:19.130 INFO:teuthology.orchestra.run.smithi131.stdout:133143986197 2024-02-19T21:50:19.130 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-02-19T21:50:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:17.465519+0000 mgr.smithi131.cjubsp (mgr.14164) 225 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:19.355 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:21.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:19.465883+0000 mgr.smithi131.cjubsp (mgr.14164) 226 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:19.465883+0000 mgr.smithi131.cjubsp (mgr.14164) 226 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:21.629 INFO:teuthology.orchestra.run.smithi131.stdout:158913789971 2024-02-19T21:50:21.630 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-02-19T21:50:21.859 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:23.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:21.466402+0000 mgr.smithi131.cjubsp (mgr.14164) 227 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:23.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:21.466402+0000 mgr.smithi131.cjubsp (mgr.14164) 227 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:24.158 INFO:teuthology.orchestra.run.smithi131.stdout:180388626446 2024-02-19T21:50:24.159 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-02-19T21:50:24.385 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:24.844 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:23.466756+0000 mgr.smithi131.cjubsp (mgr.14164) 228 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:25.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:23.466756+0000 mgr.smithi131.cjubsp (mgr.14164) 228 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:26.732 INFO:teuthology.orchestra.run.smithi131.stdout:206158430218 2024-02-19T21:50:26.733 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-02-19T21:50:26.963 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:27.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:25.467096+0000 mgr.smithi131.cjubsp (mgr.14164) 229 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:25.467096+0000 mgr.smithi131.cjubsp (mgr.14164) 229 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:28.948 INFO:teuthology.orchestra.run.smithi131.stdout:38654705705 2024-02-19T21:50:29.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:27.467608+0000 mgr.smithi131.cjubsp (mgr.14164) 230 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:27.467608+0000 mgr.smithi131.cjubsp (mgr.14164) 230 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:29.619 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705701 got 38654705705 for osd.0 2024-02-19T21:50:29.619 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-02-19T21:50:29.846 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:28.947994+0000 mon.smithi131 (mon.0) 575 : audit [DBG] from='client.? 172.21.15.131:0/336710407' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-02-19T21:50:30.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:28.947994+0000 mon.smithi131 (mon.0) 575 : audit [DBG] from='client.? 172.21.15.131:0/336710407' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-02-19T21:50:31.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:29.468068+0000 mgr.smithi131.cjubsp (mgr.14164) 231 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:31.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:29.468068+0000 mgr.smithi131.cjubsp (mgr.14164) 231 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:31.745 INFO:teuthology.orchestra.run.smithi131.stdout:60129542180 2024-02-19T21:50:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:31.746693+0000 mon.smithi131 (mon.0) 576 : audit [DBG] from='client.? 172.21.15.131:0/2677767460' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-02-19T21:50:32.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:31.746693+0000 mon.smithi131 (mon.0) 576 : audit [DBG] from='client.? 172.21.15.131:0/2677767460' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-02-19T21:50:32.400 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542176 got 60129542180 for osd.1 2024-02-19T21:50:32.400 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-02-19T21:50:32.627 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:33.103 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:31.468412+0000 mgr.smithi131.cjubsp (mgr.14164) 232 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:33.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:31.468412+0000 mgr.smithi131.cjubsp (mgr.14164) 232 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:34.383 INFO:teuthology.orchestra.run.smithi131.stdout:81604378657 2024-02-19T21:50:34.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:33.468772+0000 mgr.smithi131.cjubsp (mgr.14164) 233 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:34.953 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:34.384326+0000 mon.smithi131 (mon.0) 577 : audit [DBG] from='client.? 172.21.15.131:0/1161817780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-02-19T21:50:34.972 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378653 got 81604378657 for osd.2 2024-02-19T21:50:34.972 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-02-19T21:50:35.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:33.468772+0000 mgr.smithi131.cjubsp (mgr.14164) 233 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:35.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:34.384326+0000 mon.smithi131 (mon.0) 577 : audit [DBG] from='client.? 172.21.15.131:0/1161817780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-02-19T21:50:35.197 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:36.949 INFO:teuthology.orchestra.run.smithi131.stdout:111669149724 2024-02-19T21:50:37.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:35.469220+0000 mgr.smithi131.cjubsp (mgr.14164) 234 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:37.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:35.469220+0000 mgr.smithi131.cjubsp (mgr.14164) 234 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:37.594 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149720 got 111669149724 for osd.3 2024-02-19T21:50:37.594 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-02-19T21:50:37.818 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:38.095 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:36.950320+0000 mon.smithi131 (mon.0) 578 : audit [DBG] from='client.? 172.21.15.131:0/2568699259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-02-19T21:50:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:36.950320+0000 mon.smithi131 (mon.0) 578 : audit [DBG] from='client.? 172.21.15.131:0/2568699259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-02-19T21:50:38.898 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:37.469716+0000 mgr.smithi131.cjubsp (mgr.14164) 235 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:39.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:37.469716+0000 mgr.smithi131.cjubsp (mgr.14164) 235 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:39.592 INFO:teuthology.orchestra.run.smithi131.stdout:133143986201 2024-02-19T21:50:39.950 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:39.592874+0000 mon.smithi131 (mon.0) 579 : audit [DBG] from='client.? 172.21.15.131:0/648609456' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-02-19T21:50:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:39.592874+0000 mon.smithi131 (mon.0) 579 : audit [DBG] from='client.? 172.21.15.131:0/648609456' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-02-19T21:50:40.217 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986197 got 133143986201 for osd.4 2024-02-19T21:50:40.217 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-02-19T21:50:40.443 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:41.103 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:39.470066+0000 mgr.smithi131.cjubsp (mgr.14164) 236 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:41.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:39.470066+0000 mgr.smithi131.cjubsp (mgr.14164) 236 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:42.459 INFO:teuthology.orchestra.run.smithi131.stdout:158913789975 2024-02-19T21:50:43.078 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:41.470510+0000 mgr.smithi131.cjubsp (mgr.14164) 237 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:43.078 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:42.460471+0000 mon.smithi131 (mon.0) 580 : audit [DBG] from='client.? 172.21.15.131:0/1166147250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-02-19T21:50:43.097 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789971 got 158913789975 for osd.5 2024-02-19T21:50:43.097 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-02-19T21:50:43.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:41.470510+0000 mgr.smithi131.cjubsp (mgr.14164) 237 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:43.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:42.460471+0000 mon.smithi131 (mon.0) 580 : audit [DBG] from='client.? 172.21.15.131:0/1166147250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-02-19T21:50:43.322 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:45.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:43.470820+0000 mgr.smithi131.cjubsp (mgr.14164) 238 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:45.183 INFO:teuthology.orchestra.run.smithi131.stdout:180388626450 2024-02-19T21:50:45.211 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:43.470820+0000 mgr.smithi131.cjubsp (mgr.14164) 238 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:45.947 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626446 got 180388626450 for osd.6 2024-02-19T21:50:45.947 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-02-19T21:50:46.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:45.184420+0000 mon.smithi131 (mon.0) 581 : audit [DBG] from='client.? 172.21.15.131:0/4222124040' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-02-19T21:50:46.175 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:46.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:45.184420+0000 mon.smithi131 (mon.0) 581 : audit [DBG] from='client.? 172.21.15.131:0/4222124040' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-02-19T21:50:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:45.471137+0000 mgr.smithi131.cjubsp (mgr.14164) 239 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:46.596009+0000 mon.smithi131 (mon.0) 582 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:50:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:46.596963+0000 mon.smithi131 (mon.0) 583 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:50:47.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:46.597591+0000 mon.smithi131 (mon.0) 584 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:50:47.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:45.471137+0000 mgr.smithi131.cjubsp (mgr.14164) 239 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:47.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:46.596009+0000 mon.smithi131 (mon.0) 582 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:50:47.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:46.596963+0000 mon.smithi131 (mon.0) 583 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:50:47.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:46.597591+0000 mon.smithi131 (mon.0) 584 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:50:48.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:47.072622+0000 mon.smithi131 (mon.0) 585 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:50:48.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:47.471477+0000 mgr.smithi131.cjubsp (mgr.14164) 240 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:48.273 INFO:teuthology.orchestra.run.smithi131.stdout:206158430223 2024-02-19T21:50:48.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:47.072622+0000 mon.smithi131 (mon.0) 585 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:50:48.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:47.471477+0000 mgr.smithi131.cjubsp (mgr.14164) 240 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:48.997 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430218 got 206158430223 for osd.7 2024-02-19T21:50:48.997 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-02-19T21:50:48.997 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph pg dump --format=json 2024-02-19T21:50:49.225 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:49.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:48.274470+0000 mon.smithi131 (mon.0) 586 : audit [DBG] from='client.? 172.21.15.131:0/2194179127' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-02-19T21:50:49.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:48.274470+0000 mon.smithi131 (mon.0) 586 : audit [DBG] from='client.? 172.21.15.131:0/2194179127' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-02-19T21:50:50.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:49.471778+0000 mgr.smithi131.cjubsp (mgr.14164) 241 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:50.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:49.471778+0000 mgr.smithi131.cjubsp (mgr.14164) 241 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:51.178 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:50:51.179 INFO:teuthology.orchestra.run.smithi131.stderr:dumped all 2024-02-19T21:50:51.908 INFO:teuthology.orchestra.run.smithi131.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2024-02-19T21:50:49.471648+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":43180,"kb_used_data":3820,"kb_used_omap":0,"kb_used_meta":39296,"kb_avail":749918036,"statfs":{"total":767960285184,"available":767916068864,"internally_reserved":0,"allocated":3911680,"data_stored":1562985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40239104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.0","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2024-02-19T21:49:48.366539+0000","last_change":"2024-02-19T21:49:48.366539+0000","last_active":"2024-02-19T21:49:48.366539+0000","last_peered":"2024-02-19T21:49:48.366539+0000","last_clean":"2024-02-19T21:49:48.366539+0000","last_became_active":"2024-02-19T21:49:48.366209+0000","last_became_peered":"2024-02-19T21:49:48.366209+0000","last_unstale":"2024-02-19T21:49:48.366539+0000","last_undegraded":"2024-02-19T21:49:48.366539+0000","last_fullsized":"2024-02-19T21:49:48.366539+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-02-19T21:48:03.861851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-02-19T21:48:03.861851+0000","last_clean_scrub_stamp":"2024-02-19T21:48:03.861851+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430223,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5220,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739932,"statfs":{"total":95995035648,"available":95989690368,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67700000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69199999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626451,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5284,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739868,"statfs":{"total":95995035648,"available":95989624832,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.50600000000000001}]},{"osd":1,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56699999999999995}]},{"osd":2,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.46999999999999997}]},{"osd":3,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.52200000000000002}]},{"osd":4,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.57699999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542184,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5540,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739612,"statfs":{"total":95995035648,"available":95989362688,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:49:48 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.33500000000000002,"15min":0.33100000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.53100000000000003,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.33500000000000002,"15min":0.33000000000000002},"min":{"1min":0.25,"5min":0.222,"15min":0.222},"max":{"1min":0.60699999999999998,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.25800000000000001}]},{"osd":2,"last update":"Mon Feb 19 21:50:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.313},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.377,"15min":0.373},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.34699999999999998}]},{"osd":3,"last update":"Mon Feb 19 21:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.30099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.52600000000000002,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.41999999999999998,"15min":0.42199999999999999},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.56999999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.41999999999999998}]},{"osd":4,"last update":"Mon Feb 19 21:49:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.53100000000000003}]},{"osd":5,"last update":"Mon Feb 19 21:50:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Feb 19 21:50:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.49399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]}]},{"osd":0,"up_from":9,"seq":38654705709,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5540,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739612,"statfs":{"total":95995035648,"available":95989362688,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.317,"15min":0.308},"min":{"1min":0.25,"5min":0.16300000000000001,"15min":0.16300000000000001},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.316,"15min":0.30499999999999999},"min":{"1min":0.28899999999999998,"5min":0.193,"15min":0.193},"max":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.27100000000000002}]},{"osd":2,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.37,"15min":0.36799999999999999},"min":{"1min":0.27800000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.35199999999999998,"15min":0.34999999999999998},"min":{"1min":0.249,"5min":0.188,"15min":0.188},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.55500000000000005}]},{"osd":3,"last update":"Mon Feb 19 21:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.42199999999999999,"15min":0.42899999999999999},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.40100000000000002,"15min":0.40200000000000002},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.52700000000000002}]},{"osd":4,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.77600000000000002}]},{"osd":6,"last update":"Mon Feb 19 21:50:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.754},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.73399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72299999999999998}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5480,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739672,"statfs":{"total":95995035648,"available":95989424128,"internally_reserved":0,"allocated":491520,"data_stored":195570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:10 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.35399999999999998},"min":{"1min":0.27300000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.54400000000000004,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.35699999999999998},"min":{"1min":0.29099999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.39500000000000002}]},{"osd":1,"last update":"Mon Feb 19 21:50:10 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34399999999999997},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59499999999999997,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.38300000000000001,"15min":0.372},"min":{"1min":0.33000000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Feb 19 21:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.439,"15min":0.441},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.63500000000000001,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42299999999999999,"15min":0.42499999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.67800000000000005,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.42399999999999999}]},{"osd":4,"last update":"Mon Feb 19 21:50:44 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.59299999999999997,"15min":0.58399999999999996},"min":{"1min":0.40400000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.47299999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88600000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.622}]},{"osd":6,"last update":"Mon Feb 19 21:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.751}]},{"osd":7,"last update":"Mon Feb 19 21:50:46 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.66700000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149727,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5412,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739740,"statfs":{"total":95995035648,"available":95989493760,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":0.34899999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.215,"15min":0.215},"max":{"1min":0.45000000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.35299999999999998,"15min":0.35699999999999998},"min":{"1min":0.23799999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.42699999999999999,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.40100000000000002}]},{"osd":1,"last update":"Mon Feb 19 21:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.375,"15min":0.38100000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.438,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.377,"15min":0.38300000000000001},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.42999999999999999,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.42699999999999999}]},{"osd":2,"last update":"Mon Feb 19 21:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.42099999999999999,"15min":0.42699999999999999},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.47899999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.378,"5min":0.41399999999999998,"15min":0.41999999999999998},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.51700000000000002,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Feb 19 21:49:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.59399999999999997}]},{"osd":5,"last update":"Mon Feb 19 21:50:08 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.57899999999999996}]},{"osd":6,"last update":"Mon Feb 19 21:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.67000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5352,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739800,"statfs":{"total":95995035648,"available":95989555200,"internally_reserved":0,"allocated":491520,"data_stored":195570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45600000000000002}]},{"osd":2,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.44},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.55400000000000005}]},{"osd":3,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.30499999999999999}]},{"osd":6,"last update":"Mon Feb 19 21:50:26 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.32200000000000001}]},{"osd":7,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.39300000000000002}]}]},{"osd":5,"up_from":37,"seq":158913789976,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5352,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739800,"statfs":{"total":95995035648,"available":95989555200,"internally_reserved":0,"allocated":491520,"data_stored":195570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.502}]},{"osd":1,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.56200000000000006}]},{"osd":3,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.50900000000000001}]},{"osd":4,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.34100000000000003}]},{"osd":6,"last update":"Mon Feb 19 21:50:27 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.46700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-02-19T21:50:51.909 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph pg dump --format=json 2024-02-19T21:50:52.136 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:52.599 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:51.176598+0000 mgr.smithi131.cjubsp (mgr.14164) 242 : audit [DBG] from='client.14434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:50:52.599 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:51.472220+0000 mgr.smithi131.cjubsp (mgr.14164) 243 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:52.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:51.176598+0000 mgr.smithi131.cjubsp (mgr.14164) 242 : audit [DBG] from='client.14434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:50:52.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:51.472220+0000 mgr.smithi131.cjubsp (mgr.14164) 243 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:53.953 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:50:53.954 INFO:teuthology.orchestra.run.smithi131.stderr:dumped all 2024-02-19T21:50:54.557 INFO:teuthology.orchestra.run.smithi131.stdout:{"pg_ready":true,"pg_map":{"version":214,"stamp":"2024-02-19T21:50:53.472469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":43180,"kb_used_data":3820,"kb_used_omap":0,"kb_used_meta":39296,"kb_avail":749918036,"statfs":{"total":767960285184,"available":767916068864,"internally_reserved":0,"allocated":3911680,"data_stored":1562985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40239104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.0","version":"0'0","reported_seq":10,"reported_epoch":50,"state":"active+clean","last_fresh":"2024-02-19T21:49:48.366539+0000","last_change":"2024-02-19T21:49:48.366539+0000","last_active":"2024-02-19T21:49:48.366539+0000","last_peered":"2024-02-19T21:49:48.366539+0000","last_clean":"2024-02-19T21:49:48.366539+0000","last_became_active":"2024-02-19T21:49:48.366209+0000","last_became_peered":"2024-02-19T21:49:48.366209+0000","last_unstale":"2024-02-19T21:49:48.366539+0000","last_undegraded":"2024-02-19T21:49:48.366539+0000","last_fullsized":"2024-02-19T21:49:48.366539+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-02-19T21:48:03.861851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-02-19T21:48:03.861851+0000","last_clean_scrub_stamp":"2024-02-19T21:48:03.861851+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430224,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5220,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739932,"statfs":{"total":95995035648,"available":95989690368,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.55800000000000005}]},{"osd":1,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.59399999999999997}]},{"osd":2,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.61799999999999999}]},{"osd":3,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.57099999999999995}]},{"osd":4,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.432}]},{"osd":5,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.45700000000000002}]},{"osd":6,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.47499999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626452,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5284,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739868,"statfs":{"total":95995035648,"available":95989624832,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.55500000000000005}]},{"osd":1,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.505},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.628}]},{"osd":2,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.64600000000000002}]},{"osd":3,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.67200000000000004}]},{"osd":4,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Feb 19 21:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.66000000000000003}]},{"osd":7,"last update":"Mon Feb 19 21:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.71699999999999997}]}]},{"osd":1,"up_from":14,"seq":60129542185,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5540,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739612,"statfs":{"total":95995035648,"available":95989362688,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.34000000000000002,"15min":0.33300000000000002},"min":{"1min":0.245,"5min":0.215,"15min":0.215},"max":{"1min":0.57299999999999995,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.34599999999999997,"15min":0.33400000000000002},"min":{"1min":0.23499999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.56599999999999995,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.40100000000000002}]},{"osd":2,"last update":"Mon Feb 19 21:50:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.312},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.377,"15min":0.373},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.39000000000000001}]},{"osd":3,"last update":"Mon Feb 19 21:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.30099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.52600000000000002,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.41999999999999998,"15min":0.42199999999999999},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.56999999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.36699999999999999}]},{"osd":4,"last update":"Mon Feb 19 21:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.54000000000000004,"15min":0.52500000000000002},"min":{"1min":0.5,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.48499999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59199999999999997}]},{"osd":5,"last update":"Mon Feb 19 21:50:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.64200000000000002}]},{"osd":6,"last update":"Mon Feb 19 21:50:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.77000000000000002}]},{"osd":7,"last update":"Mon Feb 19 21:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.80300000000000005}]}]},{"osd":0,"up_from":9,"seq":38654705710,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5540,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739612,"statfs":{"total":95995035648,"available":95989362688,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Feb 19 21:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.308,"5min":0.318,"15min":0.308},"min":{"1min":0.19700000000000001,"5min":0.16300000000000001,"15min":0.16300000000000001},"max":{"1min":0.40999999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.19700000000000001},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.32300000000000001,"15min":0.307},"min":{"1min":0.22800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.45800000000000002,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.22800000000000001}]},{"osd":2,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.37,"15min":0.36799999999999999},"min":{"1min":0.27800000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.307},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.35199999999999998,"15min":0.34999999999999998},"min":{"1min":0.249,"5min":0.188,"15min":0.188},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.219}]},{"osd":3,"last update":"Mon Feb 19 21:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.42199999999999999,"15min":0.42899999999999999},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.40100000000000002,"15min":0.40200000000000002},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.28599999999999998}]},{"osd":4,"last update":"Mon Feb 19 21:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72299999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56100000000000005,"15min":0.55700000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.60199999999999998}]},{"osd":5,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Feb 19 21:50:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.63200000000000001}]},{"osd":7,"last update":"Mon Feb 19 21:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.66200000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5480,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739672,"statfs":{"total":95995035648,"available":95989424128,"internally_reserved":0,"allocated":491520,"data_stored":195570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:10 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.35899999999999999,"15min":0.35399999999999998},"min":{"1min":0.27300000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.54400000000000004,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.35699999999999998},"min":{"1min":0.29099999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.39500000000000002}]},{"osd":1,"last update":"Mon Feb 19 21:50:10 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34399999999999997},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59499999999999997,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.38300000000000001,"15min":0.372},"min":{"1min":0.33000000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Feb 19 21:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.439,"15min":0.441},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.63500000000000001,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42299999999999999,"15min":0.42499999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.67800000000000005,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.42399999999999999}]},{"osd":4,"last update":"Mon Feb 19 21:50:44 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.59299999999999997,"15min":0.58399999999999996},"min":{"1min":0.40400000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.47299999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88600000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.622}]},{"osd":6,"last update":"Mon Feb 19 21:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.751}]},{"osd":7,"last update":"Mon Feb 19 21:50:46 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.66700000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149728,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5412,"kb_used_data":476,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739740,"statfs":{"total":95995035648,"available":95989493760,"internally_reserved":0,"allocated":487424,"data_stored":195255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":0.34899999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.215,"15min":0.215},"max":{"1min":0.45000000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.35299999999999998,"15min":0.35699999999999998},"min":{"1min":0.23799999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.42699999999999999,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.41699999999999998}]},{"osd":1,"last update":"Mon Feb 19 21:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.375,"15min":0.38100000000000001},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.438,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.377,"15min":0.38300000000000001},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.42999999999999999,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.51600000000000001}]},{"osd":2,"last update":"Mon Feb 19 21:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.42099999999999999,"15min":0.42699999999999999},"min":{"1min":0.313,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.47899999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.378,"5min":0.41399999999999998,"15min":0.41999999999999998},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.51700000000000002,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.499}]},{"osd":4,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.501,"15min":0.48899999999999999},"min":{"1min":0.42499999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.76400000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.52900000000000003,"15min":0.52000000000000002},"min":{"1min":0.441,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.78200000000000003,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.67100000000000004}]},{"osd":5,"last update":"Mon Feb 19 21:50:08 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.81699999999999995}]},{"osd":6,"last update":"Mon Feb 19 21:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.75900000000000001}]},{"osd":7,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.73999999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5352,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739800,"statfs":{"total":95995035648,"available":95989555200,"internally_reserved":0,"allocated":491520,"data_stored":195570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45600000000000002}]},{"osd":2,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.44},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.55400000000000005}]},{"osd":3,"last update":"Mon Feb 19 21:49:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.30499999999999999}]},{"osd":6,"last update":"Mon Feb 19 21:50:26 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.32200000000000001}]},{"osd":7,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.39300000000000002}]}]},{"osd":5,"up_from":37,"seq":158913789977,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5352,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739800,"statfs":{"total":95995035648,"available":95989555200,"internally_reserved":0,"allocated":491520,"data_stored":195570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.59299999999999997}]},{"osd":1,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.47899999999999998}]},{"osd":2,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.61399999999999999}]},{"osd":3,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.626},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.67100000000000004}]},{"osd":4,"last update":"Mon Feb 19 21:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.40200000000000002}]},{"osd":6,"last update":"Mon Feb 19 21:50:27 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.33500000000000002}]},{"osd":7,"last update":"Mon Feb 19 21:50:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.46100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-02-19T21:50:54.558 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-02-19T21:50:54.558 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-02-19T21:50:54.558 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-02-19T21:50:54.558 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph health --format=json 2024-02-19T21:50:54.783 INFO:teuthology.orchestra.run.smithi131.stderr:Inferring config /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/mon.smithi131/config 2024-02-19T21:50:54.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:53.472712+0000 mgr.smithi131.cjubsp (mgr.14164) 244 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:54.992 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:53.951809+0000 mgr.smithi131.cjubsp (mgr.14164) 245 : audit [DBG] from='client.14438 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:50:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:53.472712+0000 mgr.smithi131.cjubsp (mgr.14164) 244 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:53.951809+0000 mgr.smithi131.cjubsp (mgr.14164) 245 : audit [DBG] from='client.14438 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:50:56.524 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:50:56.887 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:55.473096+0000 mgr.smithi131.cjubsp (mgr.14164) 246 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:56.888 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:56.524917+0000 mon.smithi131 (mon.0) 587 : audit [DBG] from='client.? 172.21.15.131:0/141563627' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-02-19T21:50:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:55.473096+0000 mgr.smithi131.cjubsp (mgr.14164) 246 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:56.524917+0000 mon.smithi131 (mon.0) 587 : audit [DBG] from='client.? 172.21.15.131:0/141563627' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-02-19T21:50:57.230 INFO:teuthology.orchestra.run.smithi131.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-02-19T21:50:57.230 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-02-19T21:50:57.230 INFO:tasks.cephadm:Setup complete, yielding 2024-02-19T21:50:57.231 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-19T21:50:57.240 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:50:57.241 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph orch status' 2024-02-19T21:50:59.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:50:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:57.473560+0000 mgr.smithi131.cjubsp (mgr.14164) 247 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:59.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:50:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:57.473560+0000 mgr.smithi131.cjubsp (mgr.14164) 247 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:50:59.277 INFO:teuthology.orchestra.run.smithi131.stdout:Backend: cephadm 2024-02-19T21:50:59.277 INFO:teuthology.orchestra.run.smithi131.stdout:Available: Yes 2024-02-19T21:50:59.277 INFO:teuthology.orchestra.run.smithi131.stdout:Paused: No 2024-02-19T21:50:59.959 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph orch ps' 2024-02-19T21:51:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:50:59.278165+0000 mgr.smithi131.cjubsp (mgr.14164) 248 : audit [DBG] from='client.14446 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:50:59.473942+0000 mgr.smithi131.cjubsp (mgr.14164) 249 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:01.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:50:59.278165+0000 mgr.smithi131.cjubsp (mgr.14164) 248 : audit [DBG] from='client.14446 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:01.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:50:59.473942+0000 mgr.smithi131.cjubsp (mgr.14164) 249 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:01.997 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:51:01.997 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (3m) 98s ago 5m 21.3M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:51:01.997 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (5m) 98s ago 5m 6387k - 16.2.5 6933c2a0b7dd 7e4833221591 2024-02-19T21:51:01.997 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (4m) 78s ago 4m 6395k - 16.2.5 6933c2a0b7dd d566245c5426 2024-02-19T21:51:01.997 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (3m) 98s ago 5m 27.4M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:51:01.997 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:9283 running (6m) 98s ago 6m 413M - 16.2.5 6933c2a0b7dd 785c7b05f4ea 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (4m) 78s ago 4m 371M - 16.2.5 6933c2a0b7dd 6776b8183cbb 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (6m) 98s ago 7m 53.6M 2048M 16.2.5 6933c2a0b7dd d880a31c6170 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (4m) 78s ago 4m 55.6M 2048M 16.2.5 6933c2a0b7dd 6a272417d66d 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (5m) 98s ago 5m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (4m) 78s ago 4m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (3m) 98s ago 3m 30.8M 3224M 16.2.5 6933c2a0b7dd f42dfded3a0d 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (3m) 98s ago 3m 32.0M 3224M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (3m) 98s ago 3m 28.2M 3224M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (2m) 98s ago 2m 30.0M 3224M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (2m) 78s ago 2m 26.5M 3992M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (2m) 78s ago 2m 28.9M 3992M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (102s) 78s ago 102s 27.3M 3992M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (81s) 78s ago 81s 14.5M 3992M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:51:01.998 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (3m) 98s ago 4m 36.2M - 2.18.1 de242295e225 22b1466d4ddf 2024-02-19T21:51:02.631 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph orch ls' 2024-02-19T21:51:03.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:01.474314+0000 mgr.smithi131.cjubsp (mgr.14164) 250 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:03.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:01.990575+0000 mgr.smithi131.cjubsp (mgr.14164) 251 : audit [DBG] from='client.14450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:01.474314+0000 mgr.smithi131.cjubsp (mgr.14164) 250 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:01.990575+0000 mgr.smithi131.cjubsp (mgr.14164) 251 : audit [DBG] from='client.14450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:04.597 INFO:teuthology.orchestra.run.smithi131.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-02-19T21:51:04.597 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager ?:9093,9094 1/1 100s ago 6m count:1 2024-02-19T21:51:04.597 INFO:teuthology.orchestra.run.smithi131.stdout:crash 2/2 100s ago 6m * 2024-02-19T21:51:04.597 INFO:teuthology.orchestra.run.smithi131.stdout:grafana ?:3000 1/1 100s ago 6m count:1 2024-02-19T21:51:04.598 INFO:teuthology.orchestra.run.smithi131.stdout:mgr 2/2 100s ago 6m count:2 2024-02-19T21:51:04.598 INFO:teuthology.orchestra.run.smithi131.stdout:mon 2/2 100s ago 5m smithi131:172.21.15.131=smithi131;smithi196:172.21.15.196=smithi196;count:2 2024-02-19T21:51:04.598 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter ?:9100 2/2 100s ago 6m * 2024-02-19T21:51:04.598 INFO:teuthology.orchestra.run.smithi131.stdout:osd 8/8 100s ago - 2024-02-19T21:51:04.598 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus ?:9095 1/1 100s ago 6m count:1 2024-02-19T21:51:04.970 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:03.474719+0000 mgr.smithi131.cjubsp (mgr.14164) 252 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:03.474719+0000 mgr.smithi131.cjubsp (mgr.14164) 252 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:05.203 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph orch host ls' 2024-02-19T21:51:05.869 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:04.595048+0000 mgr.smithi131.cjubsp (mgr.14164) 253 : audit [DBG] from='client.14454 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:05.869 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:04.853013+0000 mon.smithi131 (mon.0) 588 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:51:05.869 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:04.853987+0000 mon.smithi131 (mon.0) 589 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:51:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:04.595048+0000 mgr.smithi131.cjubsp (mgr.14164) 253 : audit [DBG] from='client.14454 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:06.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:04.853013+0000 mon.smithi131 (mon.0) 588 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:51:06.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:04.853987+0000 mon.smithi131 (mon.0) 589 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:51:06.888 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:05.475115+0000 mgr.smithi131.cjubsp (mgr.14164) 254 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:05.475115+0000 mgr.smithi131.cjubsp (mgr.14164) 254 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:07.278 INFO:teuthology.orchestra.run.smithi131.stdout:HOST ADDR LABELS STATUS 2024-02-19T21:51:07.278 INFO:teuthology.orchestra.run.smithi131.stdout:smithi131 172.21.15.131 2024-02-19T21:51:07.278 INFO:teuthology.orchestra.run.smithi131.stdout:smithi196 172.21.15.196 2024-02-19T21:51:07.951 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph orch device ls' 2024-02-19T21:51:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:07.278290+0000 mgr.smithi131.cjubsp (mgr.14164) 255 : audit [DBG] from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:07.475680+0000 mgr.smithi131.cjubsp (mgr.14164) 256 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:07.278290+0000 mgr.smithi131.cjubsp (mgr.14164) 255 : audit [DBG] from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:09.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:07.475680+0000 mgr.smithi131.cjubsp (mgr.14164) 256 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:10.158 INFO:teuthology.orchestra.run.smithi131.stdout:Hostname Path Type Serial Size Health Ident Fault Available 2024-02-19T21:51:10.158 INFO:teuthology.orchestra.run.smithi131.stdout:smithi131 /dev/nvme0n1 ssd CVFT623300N4400BGN 400G Unknown N/A N/A No 2024-02-19T21:51:10.159 INFO:teuthology.orchestra.run.smithi131.stdout:smithi196 /dev/nvme0n1 ssd CVFT6246000Z400BGN 400G Unknown N/A N/A No 2024-02-19T21:51:10.858 INFO:teuthology.run_tasks:Running task vip.exec... 2024-02-19T21:51:10.870 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:51:10.871 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-02-19T21:51:10.902 INFO:teuthology.orchestra.run.smithi131.stderr:+ systemctl stop nfs-server 2024-02-19T21:51:10.909 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi196.front.sepia.ceph.com 2024-02-19T21:51:10.909 DEBUG:teuthology.orchestra.run.smithi196:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-02-19T21:51:10.947 INFO:teuthology.orchestra.run.smithi196.stderr:+ systemctl stop nfs-server 2024-02-19T21:51:10.955 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-19T21:51:10.963 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:51:10.963 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph fs volume create foofs' 2024-02-19T21:51:11.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:09.476037+0000 mgr.smithi131.cjubsp (mgr.14164) 257 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:09.476037+0000 mgr.smithi131.cjubsp (mgr.14164) 257 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:11.810 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:10.158688+0000 mgr.smithi131.cjubsp (mgr.14164) 258 : audit [DBG] from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:10.158688+0000 mgr.smithi131.cjubsp (mgr.14164) 258 : audit [DBG] from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:13.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:11.476474+0000 mgr.smithi131.cjubsp (mgr.14164) 259 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:13.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:11.476474+0000 mgr.smithi131.cjubsp (mgr.14164) 259 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:14.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:13.115564+0000 mon.smithi131 (mon.0) 590 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-02-19T21:51:14.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:13.115564+0000 mon.smithi131 (mon.0) 590 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-02-19T21:51:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:13.115031+0000 mgr.smithi131.cjubsp (mgr.14164) 260 : audit [DBG] from='client.14466 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:13.476784+0000 mgr.smithi131.cjubsp (mgr.14164) 261 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:13.821764+0000 mon.smithi131 (mon.0) 591 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-02-19T21:51:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:13.821813+0000 mon.smithi131 (mon.0) 592 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-02-19T21:51:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:13.822679+0000 mon.smithi131 (mon.0) 593 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-02-19T21:51:15.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:51:14.829+0000 7f82a01ed700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:51:15.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:13.115031+0000 mgr.smithi131.cjubsp (mgr.14164) 260 : audit [DBG] from='client.14466 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:15.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:13.476784+0000 mgr.smithi131.cjubsp (mgr.14164) 261 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:15.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:13.821764+0000 mon.smithi131 (mon.0) 591 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-02-19T21:51:15.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:13.821813+0000 mon.smithi131 (mon.0) 592 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-02-19T21:51:15.196 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:13.822679+0000 mon.smithi131 (mon.0) 593 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-02-19T21:51:15.529 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-02-19T21:51:15.538 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-02-19T21:51:15.538 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch ls -f json 2024-02-19T21:51:16.089 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.824550+0000 mon.smithi131 (mon.0) 594 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-02-19T21:51:16.089 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:14.824744+0000 mon.smithi131 (mon.0) 595 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.829799+0000 mon.smithi131 (mon.0) 596 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:14.830821+0000 mon.smithi131 (mon.0) 597 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:14.830852+0000 mon.smithi131 (mon.0) 598 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:14.848092+0000 mon.smithi131 (mon.0) 599 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.848201+0000 mon.smithi131 (mon.0) 600 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:14.848276+0000 mon.smithi131 (mon.0) 601 : cluster [DBG] fsmap foofs:0 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:14.850671+0000 mgr.smithi131.cjubsp (mgr.14164) 262 : cephadm [INF] Saving service mds.foofs spec with placement count:2 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.854247+0000 mon.smithi131 (mon.0) 602 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.854993+0000 mon.smithi131 (mon.0) 603 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.857421+0000 mon.smithi131 (mon.0) 604 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.858439+0000 mon.smithi131 (mon.0) 605 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.866889+0000 mon.smithi131 (mon.0) 606 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.877010+0000 mon.smithi131 (mon.0) 607 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:16.090 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.877891+0000 mon.smithi131 (mon.0) 608 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi131.gemwnz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-19T21:51:16.091 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.884756+0000 mon.smithi131 (mon.0) 609 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi131.gemwnz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-02-19T21:51:16.091 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:14.885634+0000 mon.smithi131 (mon.0) 610 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:16.091 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:14.886419+0000 mgr.smithi131.cjubsp (mgr.14164) 263 : cephadm [INF] Deploying daemon mds.foofs.smithi131.gemwnz on smithi131 2024-02-19T21:51:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.824550+0000 mon.smithi131 (mon.0) 594 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-02-19T21:51:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:14.824744+0000 mon.smithi131 (mon.0) 595 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.829799+0000 mon.smithi131 (mon.0) 596 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:14.830821+0000 mon.smithi131 (mon.0) 597 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:14.830852+0000 mon.smithi131 (mon.0) 598 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:14.848092+0000 mon.smithi131 (mon.0) 599 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.848201+0000 mon.smithi131 (mon.0) 600 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:14.848276+0000 mon.smithi131 (mon.0) 601 : cluster [DBG] fsmap foofs:0 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:14.850671+0000 mgr.smithi131.cjubsp (mgr.14164) 262 : cephadm [INF] Saving service mds.foofs spec with placement count:2 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.854247+0000 mon.smithi131 (mon.0) 602 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.854993+0000 mon.smithi131 (mon.0) 603 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.857421+0000 mon.smithi131 (mon.0) 604 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.858439+0000 mon.smithi131 (mon.0) 605 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.866889+0000 mon.smithi131 (mon.0) 606 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.877010+0000 mon.smithi131 (mon.0) 607 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:16.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.877891+0000 mon.smithi131 (mon.0) 608 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi131.gemwnz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-19T21:51:16.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.884756+0000 mon.smithi131 (mon.0) 609 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi131.gemwnz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-02-19T21:51:16.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:14.885634+0000 mon.smithi131 (mon.0) 610 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:16.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:14.886419+0000 mgr.smithi131.cjubsp (mgr.14164) 263 : cephadm [INF] Deploying daemon mds.foofs.smithi131.gemwnz on smithi131 2024-02-19T21:51:17.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:15.477352+0000 mgr.smithi131.cjubsp (mgr.14164) 264 : cluster [DBG] pgmap v228: 65 pgs: 9 creating+peering, 6 active+clean, 50 unknown; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:17.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:15.851883+0000 mon.smithi131 (mon.0) 611 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-02-19T21:51:17.192 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:15.477352+0000 mgr.smithi131.cjubsp (mgr.14164) 264 : cluster [DBG] pgmap v228: 65 pgs: 9 creating+peering, 6 active+clean, 50 unknown; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:17.192 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:15.851883+0000 mon.smithi131 (mon.0) 611 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-02-19T21:51:18.137 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:16.851757+0000 mon.smithi131 (mon.0) 612 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-02-19T21:51:18.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:16.851757+0000 mon.smithi131 (mon.0) 612 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-02-19T21:51:18.687 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:51:18.687 INFO:teuthology.orchestra.run.smithi131.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-02-19T21:44:53.589413Z", "last_refresh": "2024-02-19T21:49:23.857244Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-02-19T21:44:40.820307Z", "last_refresh": "2024-02-19T21:49:23.857299Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-02-19T21:44:50.364803Z", "last_refresh": "2024-02-19T21:49:23.857353Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-02-19T21:51:14.854435Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-02-19T21:51:14.850696Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-02-19T21:44:39.551937Z", "last_refresh": "2024-02-19T21:49:23.857182Z", "running": 2, "size": 2}}, {"events": ["2024-02-19T21:45:52.406412Z service:mon [INFO] \"service was created\""], "placement": {"count": 2, "hosts": ["smithi131:172.21.15.131=smithi131", "smithi196:172.21.15.196=smithi196"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-02-19T21:45:52.401306Z", "last_refresh": "2024-02-19T21:49:23.857035Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-02-19T21:44:51.697757Z", "last_refresh": "2024-02-19T21:49:23.857406Z", "ports": [9100], "running": 2, "size": 2}}, {"placement": {}, "service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355", "container_image_name": "docker.io/ceph/ceph@sha256:829ebf54704f2d827de00913b171e5da741aad9b53c1f35ad59251524790eceb", "last_refresh": "2024-02-19T21:49:23.857512Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-02-19T21:44:49.047885Z", "last_refresh": "2024-02-19T21:49:23.857459Z", "ports": [9095], "running": 1, "size": 1}}] 2024-02-19T21:51:19.172 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:17.478032+0000 mgr.smithi131.cjubsp (mgr.14164) 265 : cluster [DBG] pgmap v231: 65 pgs: 1 creating+activating, 11 creating+peering, 48 active+clean, 5 unknown; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:19.172 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:18.706995+0000 mon.smithi131 (mon.0) 613 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:18.707966+0000 mon.smithi131 (mon.0) 614 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi196.hhhkwt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:18.712712+0000 mon.smithi131 (mon.0) 615 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi196.hhhkwt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:18.713257+0000 mon.smithi131 (mon.0) 616 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:18.777650+0000 mon.smithi131 (mon.0) 617 : cluster [INF] daemon mds.foofs.smithi131.gemwnz assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:18.777770+0000 mon.smithi131 (mon.0) 618 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:18.777808+0000 mon.smithi131 (mon.0) 619 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:18.777833+0000 mon.smithi131 (mon.0) 620 : cluster [INF] Cluster is now healthy 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:18.781870+0000 mon.smithi131 (mon.0) 621 : cluster [DBG] mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639] up:boot 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:18.781924+0000 mon.smithi131 (mon.0) 622 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:creating} 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:18.782400+0000 mon.smithi131 (mon.0) 623 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:51:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:18.796841+0000 mon.smithi131 (mon.0) 624 : cluster [INF] daemon mds.foofs.smithi131.gemwnz is now active in filesystem foofs as rank 0 2024-02-19T21:51:19.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:17.478032+0000 mgr.smithi131.cjubsp (mgr.14164) 265 : cluster [DBG] pgmap v231: 65 pgs: 1 creating+activating, 11 creating+peering, 48 active+clean, 5 unknown; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:19.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:18.706995+0000 mon.smithi131 (mon.0) 613 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:19.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:18.707966+0000 mon.smithi131 (mon.0) 614 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi196.hhhkwt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:18.712712+0000 mon.smithi131 (mon.0) 615 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi196.hhhkwt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:18.713257+0000 mon.smithi131 (mon.0) 616 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:18.777650+0000 mon.smithi131 (mon.0) 617 : cluster [INF] daemon mds.foofs.smithi131.gemwnz assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:18.777770+0000 mon.smithi131 (mon.0) 618 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:18.777808+0000 mon.smithi131 (mon.0) 619 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:18.777833+0000 mon.smithi131 (mon.0) 620 : cluster [INF] Cluster is now healthy 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:18.781870+0000 mon.smithi131 (mon.0) 621 : cluster [DBG] mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639] up:boot 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:18.781924+0000 mon.smithi131 (mon.0) 622 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:creating} 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:18.782400+0000 mon.smithi131 (mon.0) 623 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:51:19.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:18.796841+0000 mon.smithi131 (mon.0) 624 : cluster [INF] daemon mds.foofs.smithi131.gemwnz is now active in filesystem foofs as rank 0 2024-02-19T21:51:19.358 INFO:tasks.cephadm:mds.foofs has 0/2 2024-02-19T21:51:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:18.681479+0000 mgr.smithi131.cjubsp (mgr.14164) 266 : audit [DBG] from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:51:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:18.713718+0000 mgr.smithi131.cjubsp (mgr.14164) 267 : cephadm [INF] Deploying daemon mds.foofs.smithi196.hhhkwt on smithi196 2024-02-19T21:51:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:19.791375+0000 mon.smithi131 (mon.0) 625 : cluster [DBG] mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639] up:active 2024-02-19T21:51:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:19.791456+0000 mon.smithi131 (mon.0) 626 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 2024-02-19T21:51:20.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:18.681479+0000 mgr.smithi131.cjubsp (mgr.14164) 266 : audit [DBG] from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:51:20.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:18.713718+0000 mgr.smithi131.cjubsp (mgr.14164) 267 : cephadm [INF] Deploying daemon mds.foofs.smithi196.hhhkwt on smithi196 2024-02-19T21:51:20.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:19.791375+0000 mon.smithi131 (mon.0) 625 : cluster [DBG] mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639] up:active 2024-02-19T21:51:20.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:19.791456+0000 mon.smithi131 (mon.0) 626 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 2024-02-19T21:51:20.359 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch ls -f json 2024-02-19T21:51:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:19.478583+0000 mgr.smithi131.cjubsp (mgr.14164) 268 : cluster [DBG] pgmap v232: 65 pgs: 1 creating+activating, 7 creating+peering, 57 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:19.478583+0000 mgr.smithi131.cjubsp (mgr.14164) 268 : cluster [DBG] pgmap v232: 65 pgs: 1 creating+activating, 7 creating+peering, 57 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-02-19T21:51:22.326 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:51:22.326 INFO:teuthology.orchestra.run.smithi131.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-02-19T21:44:53.589413Z", "last_refresh": "2024-02-19T21:49:23.857244Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-02-19T21:44:40.820307Z", "last_refresh": "2024-02-19T21:49:23.857299Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-02-19T21:44:50.364803Z", "last_refresh": "2024-02-19T21:49:23.857353Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-02-19T21:51:14.854435Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-02-19T21:51:14.850696Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-02-19T21:44:39.551937Z", "last_refresh": "2024-02-19T21:49:23.857182Z", "running": 2, "size": 2}}, {"events": ["2024-02-19T21:45:52.406412Z service:mon [INFO] \"service was created\""], "placement": {"count": 2, "hosts": ["smithi131:172.21.15.131=smithi131", "smithi196:172.21.15.196=smithi196"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-02-19T21:45:52.401306Z", "last_refresh": "2024-02-19T21:49:23.857035Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-02-19T21:44:51.697757Z", "last_refresh": "2024-02-19T21:49:23.857406Z", "ports": [9100], "running": 2, "size": 2}}, {"placement": {}, "service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "6933c2a0b7ddc222e77dc8d9dc471a0c639a1c0bded5077be53ad3a9557b4355", "container_image_name": "docker.io/ceph/ceph@sha256:829ebf54704f2d827de00913b171e5da741aad9b53c1f35ad59251524790eceb", "last_refresh": "2024-02-19T21:49:23.857512Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-02-19T21:44:49.047885Z", "last_refresh": "2024-02-19T21:49:23.857459Z", "ports": [9095], "running": 1, "size": 1}}] 2024-02-19T21:51:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:21.136592+0000 mon.smithi131 (mon.0) 627 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:21.138077+0000 mon.smithi131 (mon.0) 628 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:21.138960+0000 mon.smithi131 (mon.0) 629 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:21.139506+0000 mon.smithi131 (mon.0) 630 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:21.479041+0000 mgr.smithi131.cjubsp (mgr.14164) 269 : cluster [DBG] pgmap v233: 65 pgs: 65 active+clean; 1.5 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 7 op/s 2024-02-19T21:51:22.460 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:21.136592+0000 mon.smithi131 (mon.0) 627 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:22.460 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:21.138077+0000 mon.smithi131 (mon.0) 628 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:22.460 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:21.138960+0000 mon.smithi131 (mon.0) 629 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:22.460 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:21.139506+0000 mon.smithi131 (mon.0) 630 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:22.460 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:21.479041+0000 mgr.smithi131.cjubsp (mgr.14164) 269 : cluster [DBG] pgmap v233: 65 pgs: 65 active+clean; 1.5 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 7 op/s 2024-02-19T21:51:23.080 INFO:tasks.cephadm:mds.foofs has 2/2 2024-02-19T21:51:23.080 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-19T21:51:23.089 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:51:23.090 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-02-19T21:51:23.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:22.323902+0000 mgr.smithi131.cjubsp (mgr.14164) 270 : audit [DBG] from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:51:23.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:22.323902+0000 mgr.smithi131.cjubsp (mgr.14164) 270 : audit [DBG] from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T21:51:24.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:23.223822+0000 mon.smithi131 (mon.0) 631 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:24.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:23.479623+0000 mgr.smithi131.cjubsp (mgr.14164) 271 : cluster [DBG] pgmap v234: 65 pgs: 65 active+clean; 2.2 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s wr, 6 op/s 2024-02-19T21:51:24.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:23.223822+0000 mon.smithi131 (mon.0) 631 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:24.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:23.479623+0000 mgr.smithi131.cjubsp (mgr.14164) 271 : cluster [DBG] pgmap v234: 65 pgs: 65 active+clean; 2.2 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s wr, 6 op/s 2024-02-19T21:51:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:24.875529+0000 mon.smithi131 (mon.0) 632 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:24.875529+0000 mon.smithi131 (mon.0) 632 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:25.480242+0000 mgr.smithi131.cjubsp (mgr.14164) 272 : cluster [DBG] pgmap v235: 65 pgs: 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 4 op/s 2024-02-19T21:51:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:25.880669+0000 mon.smithi131 (mon.0) 633 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:boot 2024-02-19T21:51:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:25.880956+0000 mon.smithi131 (mon.0) 634 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:51:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:25.881071+0000 mon.smithi131 (mon.0) 635 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:51:27.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:25.885023+0000 mon.smithi131 (mon.0) 636 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:51:27.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:26.354259+0000 mon.smithi131 (mon.0) 637 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]: dispatch 2024-02-19T21:51:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:25.480242+0000 mgr.smithi131.cjubsp (mgr.14164) 272 : cluster [DBG] pgmap v235: 65 pgs: 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 4 op/s 2024-02-19T21:51:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:25.880669+0000 mon.smithi131 (mon.0) 633 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:boot 2024-02-19T21:51:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:25.880956+0000 mon.smithi131 (mon.0) 634 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:51:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:25.881071+0000 mon.smithi131 (mon.0) 635 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:51:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:25.885023+0000 mon.smithi131 (mon.0) 636 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:51:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:26.354259+0000 mon.smithi131 (mon.0) 637 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]: dispatch 2024-02-19T21:51:28.073 INFO:teuthology.orchestra.run.smithi131.stdout:NFS Cluster Created Successfully 2024-02-19T21:51:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:26.353091+0000 mgr.smithi131.cjubsp (mgr.14164) 273 : audit [DBG] from='client.14486 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "clusterid": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:27.042022+0000 mon.smithi131 (mon.0) 638 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]': finished 2024-02-19T21:51:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:27.042074+0000 mon.smithi131 (mon.0) 639 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-02-19T21:51:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:27.043214+0000 mon.smithi131 (mon.0) 640 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]: dispatch 2024-02-19T21:51:28.435 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:26.353091+0000 mgr.smithi131.cjubsp (mgr.14164) 273 : audit [DBG] from='client.14486 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "clusterid": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:28.435 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:27.042022+0000 mon.smithi131 (mon.0) 638 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]': finished 2024-02-19T21:51:28.435 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:27.042074+0000 mon.smithi131 (mon.0) 639 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-02-19T21:51:28.436 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:27.043214+0000 mon.smithi131 (mon.0) 640 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]: dispatch 2024-02-19T21:51:28.710 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-02-19T21:51:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:27.480575+0000 mgr.smithi131.cjubsp (mgr.14164) 274 : cluster [DBG] pgmap v237: 97 pgs: 32 unknown, 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 3 op/s 2024-02-19T21:51:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.047720+0000 mon.smithi131 (mon.0) 641 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]': finished 2024-02-19T21:51:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:28.047962+0000 mon.smithi131 (mon.0) 642 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-02-19T21:51:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.074152+0000 mon.smithi131 (mon.0) 643 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.869091+0000 mon.smithi131 (mon.0) 644 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.872927+0000 mon.smithi131 (mon.0) 645 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.878922+0000 mon.smithi131 (mon.0) 646 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.889750+0000 mon.smithi131 (mon.0) 647 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi131.nauhyt", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.893249+0000 mon.smithi131 (mon.0) 648 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi131.nauhyt", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.894127+0000 mon.smithi131 (mon.0) 649 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]: dispatch 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.897192+0000 mon.smithi131 (mon.0) 650 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]': finished 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.898235+0000 mon.smithi131 (mon.0) 651 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.942256+0000 mon.smithi131 (mon.0) 652 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.945963+0000 mon.smithi131 (mon.0) 653 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.989887+0000 mon.smithi131 (mon.0) 654 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi131.nauhyt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.993389+0000 mon.smithi131 (mon.0) 655 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi131.nauhyt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-19T21:51:29.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:28.997009+0000 mon.smithi131 (mon.0) 656 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:27.480575+0000 mgr.smithi131.cjubsp (mgr.14164) 274 : cluster [DBG] pgmap v237: 97 pgs: 32 unknown, 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 3 op/s 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.047720+0000 mon.smithi131 (mon.0) 641 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]': finished 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:28.047962+0000 mon.smithi131 (mon.0) 642 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.074152+0000 mon.smithi131 (mon.0) 643 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.869091+0000 mon.smithi131 (mon.0) 644 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.872927+0000 mon.smithi131 (mon.0) 645 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.878922+0000 mon.smithi131 (mon.0) 646 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.889750+0000 mon.smithi131 (mon.0) 647 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi131.nauhyt", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.893249+0000 mon.smithi131 (mon.0) 648 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi131.nauhyt", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2024-02-19T21:51:29.448 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.894127+0000 mon.smithi131 (mon.0) 649 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]: dispatch 2024-02-19T21:51:29.449 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.897192+0000 mon.smithi131 (mon.0) 650 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]': finished 2024-02-19T21:51:29.449 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.898235+0000 mon.smithi131 (mon.0) 651 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:29.449 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.942256+0000 mon.smithi131 (mon.0) 652 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:51:29.449 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.945963+0000 mon.smithi131 (mon.0) 653 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:51:29.449 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.989887+0000 mon.smithi131 (mon.0) 654 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi131.nauhyt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:51:29.449 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.993389+0000 mon.smithi131 (mon.0) 655 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi131.nauhyt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-19T21:51:29.449 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:28.997009+0000 mon.smithi131 (mon.0) 656 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:30.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:28.070994+0000 mgr.smithi131.cjubsp (mgr.14164) 275 : cephadm [INF] Saving service nfs.foo spec with placement count:2 2024-02-19T21:51:30.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:28.889458+0000 mgr.smithi131.cjubsp (mgr.14164) 276 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi131.nauhyt 2024-02-19T21:51:30.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:28.893830+0000 mgr.smithi131.cjubsp (mgr.14164) 277 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-19T21:51:30.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:28.989471+0000 mgr.smithi131.cjubsp (mgr.14164) 278 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:51:30.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:28.989562+0000 mgr.smithi131.cjubsp (mgr.14164) 279 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi131.nauhyt-rgw 2024-02-19T21:51:30.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:28.997693+0000 mgr.smithi131.cjubsp (mgr.14164) 280 : cephadm [INF] Deploying daemon nfs.foo.0.0.smithi131.nauhyt on smithi131 2024-02-19T21:51:30.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:29.049009+0000 mon.smithi131 (mon.0) 657 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-02-19T21:51:30.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:29.480949+0000 mgr.smithi131.cjubsp (mgr.14164) 281 : cluster [DBG] pgmap v240: 97 pgs: 22 unknown, 75 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-02-19T21:51:30.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:29.994678+0000 mon.smithi131 (mon.0) 658 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:standby 2024-02-19T21:51:30.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:29.994721+0000 mon.smithi131 (mon.0) 659 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:28.070994+0000 mgr.smithi131.cjubsp (mgr.14164) 275 : cephadm [INF] Saving service nfs.foo spec with placement count:2 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:28.889458+0000 mgr.smithi131.cjubsp (mgr.14164) 276 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi131.nauhyt 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:28.893830+0000 mgr.smithi131.cjubsp (mgr.14164) 277 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:28.989471+0000 mgr.smithi131.cjubsp (mgr.14164) 278 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:28.989562+0000 mgr.smithi131.cjubsp (mgr.14164) 279 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi131.nauhyt-rgw 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:28.997693+0000 mgr.smithi131.cjubsp (mgr.14164) 280 : cephadm [INF] Deploying daemon nfs.foo.0.0.smithi131.nauhyt on smithi131 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:29.049009+0000 mon.smithi131 (mon.0) 657 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:29.480949+0000 mgr.smithi131.cjubsp (mgr.14164) 281 : cluster [DBG] pgmap v240: 97 pgs: 22 unknown, 75 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:29.994678+0000 mon.smithi131 (mon.0) 658 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:standby 2024-02-19T21:51:30.447 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:29.994721+0000 mon.smithi131 (mon.0) 659 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:51:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:30.062309+0000 mon.smithi131 (mon.0) 660 : cluster [DBG] mgrmap e19: smithi131.cjubsp(active, since 6m), standbys: smithi196.voprex 2024-02-19T21:51:31.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:30.062309+0000 mon.smithi131 (mon.0) 660 : cluster [DBG] mgrmap e19: smithi131.cjubsp(active, since 6m), standbys: smithi196.voprex 2024-02-19T21:51:31.525 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:51:31.526 INFO:teuthology.orchestra.run.smithi131.stdout: "bind": "/fake", 2024-02-19T21:51:31.526 INFO:teuthology.orchestra.run.smithi131.stdout: "fs": "foofs", 2024-02-19T21:51:31.526 INFO:teuthology.orchestra.run.smithi131.stdout: "path": "/", 2024-02-19T21:51:31.526 INFO:teuthology.orchestra.run.smithi131.stdout: "cluster": "foo", 2024-02-19T21:51:31.526 INFO:teuthology.orchestra.run.smithi131.stdout: "mode": "RW" 2024-02-19T21:51:31.526 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:51:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:31.481351+0000 mgr.smithi131.cjubsp (mgr.14164) 282 : cluster [DBG] pgmap v241: 97 pgs: 17 unknown, 80 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-02-19T21:51:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:31.500869+0000 mgr.smithi131.cjubsp (mgr.14164) 283 : audit [DBG] from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "clusterid": "foo", "binding": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:31.519901+0000 mon.smithi131 (mon.0) 661 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-02-19T21:51:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:31.523127+0000 mon.smithi131 (mon.0) 662 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-02-19T21:51:32.320 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-02-19T21:51:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:31.481351+0000 mgr.smithi131.cjubsp (mgr.14164) 282 : cluster [DBG] pgmap v241: 97 pgs: 17 unknown, 80 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-02-19T21:51:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:31.500869+0000 mgr.smithi131.cjubsp (mgr.14164) 283 : audit [DBG] from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "clusterid": "foo", "binding": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:31.519901+0000 mon.smithi131 (mon.0) 661 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-02-19T21:51:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:31.523127+0000 mon.smithi131 (mon.0) 662 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-02-19T21:51:34.306 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:33.166366+0000 mon.smithi131 (mon.0) 663 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:34.306 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:33.167291+0000 mgr.smithi131.cjubsp (mgr.14164) 284 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi196.hmjmsf 2024-02-19T21:51:34.307 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:33.167478+0000 mon.smithi131 (mon.0) 664 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2024-02-19T21:51:34.307 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:33.170407+0000 mon.smithi131 (mon.0) 665 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2024-02-19T21:51:34.307 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:33.170872+0000 mgr.smithi131.cjubsp (mgr.14164) 285 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-19T21:51:34.307 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:33.171127+0000 mon.smithi131 (mon.0) 666 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]: dispatch 2024-02-19T21:51:34.307 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:33.175706+0000 mon.smithi131 (mon.0) 667 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]': finished 2024-02-19T21:51:34.307 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:33.176700+0000 mon.smithi131 (mon.0) 668 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:34.307 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:33.481923+0000 mgr.smithi131.cjubsp (mgr.14164) 286 : cluster [DBG] pgmap v242: 97 pgs: 97 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 158 B/s rd, 476 B/s wr, 0 op/s 2024-02-19T21:51:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:33.166366+0000 mon.smithi131 (mon.0) 663 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:33.167291+0000 mgr.smithi131.cjubsp (mgr.14164) 284 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi196.hmjmsf 2024-02-19T21:51:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:33.167478+0000 mon.smithi131 (mon.0) 664 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2024-02-19T21:51:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:33.170407+0000 mon.smithi131 (mon.0) 665 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2024-02-19T21:51:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:33.170872+0000 mgr.smithi131.cjubsp (mgr.14164) 285 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-19T21:51:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:33.171127+0000 mon.smithi131 (mon.0) 666 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]: dispatch 2024-02-19T21:51:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:33.175706+0000 mon.smithi131 (mon.0) 667 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool nfs-ganesha"]}]': finished 2024-02-19T21:51:34.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:33.176700+0000 mon.smithi131 (mon.0) 668 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:34.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:33.481923+0000 mgr.smithi131.cjubsp (mgr.14164) 286 : cluster [DBG] pgmap v242: 97 pgs: 97 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 158 B/s rd, 476 B/s wr, 0 op/s 2024-02-19T21:51:35.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:34.699309+0000 mgr.smithi131.cjubsp (mgr.14164) 287 : audit [DBG] from='client.14520 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:35.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:34.699309+0000 mgr.smithi131.cjubsp (mgr.14164) 287 : audit [DBG] from='client.14520 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:35.482349+0000 mgr.smithi131.cjubsp (mgr.14164) 288 : cluster [DBG] pgmap v243: 97 pgs: 97 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 383 B/s wr, 0 op/s 2024-02-19T21:51:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:35.985295+0000 mgr.smithi131.cjubsp (mgr.14164) 289 : audit [DBG] from='client.14524 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:36.071843+0000 mon.smithi131 (mon.0) 669 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:51:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:36.074953+0000 mon.smithi131 (mon.0) 670 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:51:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:36.122666+0000 mon.smithi131 (mon.0) 671 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:51:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:36.125685+0000 mon.smithi131 (mon.0) 672 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-19T21:51:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:36.126896+0000 mon.smithi131 (mon.0) 673 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:36.604 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:35.482349+0000 mgr.smithi131.cjubsp (mgr.14164) 288 : cluster [DBG] pgmap v243: 97 pgs: 97 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 383 B/s wr, 0 op/s 2024-02-19T21:51:36.604 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:35.985295+0000 mgr.smithi131.cjubsp (mgr.14164) 289 : audit [DBG] from='client.14524 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:36.604 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:36.071843+0000 mon.smithi131 (mon.0) 669 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:51:36.604 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:36.074953+0000 mon.smithi131 (mon.0) 670 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:51:36.604 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:36.122666+0000 mon.smithi131 (mon.0) 671 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:51:36.604 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:36.125685+0000 mon.smithi131 (mon.0) 672 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-19T21:51:36.604 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:36.126896+0000 mon.smithi131 (mon.0) 673 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:37.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:36.122237+0000 mgr.smithi131.cjubsp (mgr.14164) 290 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:51:37.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:36.122340+0000 mgr.smithi131.cjubsp (mgr.14164) 291 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi196.hmjmsf-rgw 2024-02-19T21:51:37.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:51:36.127514+0000 mgr.smithi131.cjubsp (mgr.14164) 292 : cephadm [INF] Deploying daemon nfs.foo.1.0.smithi196.hmjmsf on smithi196 2024-02-19T21:51:37.591 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:36.122237+0000 mgr.smithi131.cjubsp (mgr.14164) 290 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:51:37.591 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:36.122340+0000 mgr.smithi131.cjubsp (mgr.14164) 291 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi196.hmjmsf-rgw 2024-02-19T21:51:37.591 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:51:36.127514+0000 mgr.smithi131.cjubsp (mgr.14164) 292 : cephadm [INF] Deploying daemon nfs.foo.1.0.smithi196.hmjmsf on smithi196 2024-02-19T21:51:38.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:37.260325+0000 mgr.smithi131.cjubsp (mgr.14164) 293 : audit [DBG] from='client.14538 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:38.446 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:37.482901+0000 mgr.smithi131.cjubsp (mgr.14164) 294 : cluster [DBG] pgmap v244: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-02-19T21:51:38.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:37.260325+0000 mgr.smithi131.cjubsp (mgr.14164) 293 : audit [DBG] from='client.14538 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:37.482901+0000 mgr.smithi131.cjubsp (mgr.14164) 294 : cluster [DBG] pgmap v244: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-02-19T21:51:38.573 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo 2/2 - 10s count:2 2024-02-19T21:51:39.183 INFO:teuthology.run_tasks:Running task vip.exec... 2024-02-19T21:51:39.193 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:51:39.193 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-02-19T21:51:39.225 INFO:teuthology.orchestra.run.smithi131.stderr:+ mkdir /mnt/foo 2024-02-19T21:51:39.249 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-02-19T21:51:39.317 INFO:teuthology.orchestra.run.smithi131.stderr:++ hostname 2024-02-19T21:51:39.318 INFO:teuthology.orchestra.run.smithi131.stderr:+ mount -t nfs smithi131:/fake /mnt/foo -o sync 2024-02-19T21:51:39.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:38.505582+0000 mon.smithi131 (mon.0) 674 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:39.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:38.506944+0000 mon.smithi131 (mon.0) 675 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:39.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:38.508087+0000 mon.smithi131 (mon.0) 676 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:39.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:38.508840+0000 mon.smithi131 (mon.0) 677 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:39.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:38.558505+0000 mgr.smithi131.cjubsp (mgr.14164) 295 : audit [DBG] from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:39.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:38.505582+0000 mon.smithi131 (mon.0) 674 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:39.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:38.506944+0000 mon.smithi131 (mon.0) 675 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:39.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:38.508087+0000 mon.smithi131 (mon.0) 676 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:39.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:38.508840+0000 mon.smithi131 (mon.0) 677 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:39.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:38.558505+0000 mgr.smithi131.cjubsp (mgr.14164) 295 : audit [DBG] from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:51:40.011 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-02-19T21:51:40.084 INFO:teuthology.orchestra.run.smithi131.stderr:+ echo test 2024-02-19T21:51:40.091 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-02-19T21:51:40.157 INFO:teuthology.orchestra.run.smithi131.stderr:+ sync 2024-02-19T21:51:40.731 INFO:teuthology.run_tasks:Running task parallel... 2024-02-19T21:51:40.741 INFO:teuthology.task.parallel:starting parallel... 2024-02-19T21:51:40.742 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-02-19T21:51:40.742 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-02-19T21:51:40.743 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:51:40.743 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-02-19T21:51:40.743 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-02-19T21:51:40.743 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-02-19T21:51:40.744 INFO:teuthology.task.exec:Executing custom commands... 2024-02-19T21:51:40.744 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:51:40.744 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-02-19T21:51:40.872 INFO:teuthology.orchestra.run.smithi131.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-02-19T21:51:40.873 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:51:40.873 INFO:teuthology.orchestra.run.smithi131.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-02-19T21:51:40.894 INFO:teuthology.orchestra.run.smithi131.stdout:4 of 5 processes prepared for launch 0 sec 2024-02-19T21:51:40.994 INFO:teuthology.orchestra.run.smithi131.stdout:5 of 5 processes prepared for launch 0 sec 2024-02-19T21:51:40.994 INFO:teuthology.orchestra.run.smithi131.stdout:releasing clients 2024-02-19T21:51:41.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:39.483289+0000 mgr.smithi131.cjubsp (mgr.14164) 296 : cluster [DBG] pgmap v245: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-02-19T21:51:41.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:39.881878+0000 mon.smithi131 (mon.0) 678 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:41.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:39.483289+0000 mgr.smithi131.cjubsp (mgr.14164) 296 : cluster [DBG] pgmap v245: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-02-19T21:51:41.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:39.881878+0000 mon.smithi131 (mon.0) 678 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:41.994 INFO:teuthology.orchestra.run.smithi131.stdout: 5 312 57.17 MB/sec warmup 1 sec latency 21.538 ms 2024-02-19T21:51:42.658 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:41.415845+0000 mon.smithi131 (mon.0) 679 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:42.658 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:41.483877+0000 mgr.smithi131.cjubsp (mgr.14164) 297 : cluster [DBG] pgmap v246: 97 pgs: 97 active+clean; 203 KiB data, 46 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 19 KiB/s wr, 4 op/s 2024-02-19T21:51:42.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:41.415845+0000 mon.smithi131 (mon.0) 679 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:42.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:41.483877+0000 mgr.smithi131.cjubsp (mgr.14164) 297 : cluster [DBG] pgmap v246: 97 pgs: 97 active+clean; 203 KiB data, 46 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 19 KiB/s wr, 4 op/s 2024-02-19T21:51:42.994 INFO:teuthology.orchestra.run.smithi131.stdout: 5 703 53.04 MB/sec warmup 2 sec latency 11.149 ms 2024-02-19T21:51:43.994 INFO:teuthology.orchestra.run.smithi131.stdout: 5 1928 44.05 MB/sec warmup 3 sec latency 12.558 ms 2024-02-19T21:51:44.532 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-02-19T21:51:44.994 INFO:teuthology.orchestra.run.smithi131.stdout: 5 3367 44.97 MB/sec warmup 4 sec latency 9.128 ms 2024-02-19T21:51:45.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:43.484439+0000 mgr.smithi131.cjubsp (mgr.14164) 298 : cluster [DBG] pgmap v247: 97 pgs: 97 active+clean; 24 MiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.1 MiB/s wr, 44 op/s 2024-02-19T21:51:45.207 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:43.484439+0000 mgr.smithi131.cjubsp (mgr.14164) 298 : cluster [DBG] pgmap v247: 97 pgs: 97 active+clean; 24 MiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.1 MiB/s wr, 44 op/s 2024-02-19T21:51:45.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 4494 40.89 MB/sec warmup 5 sec latency 11.437 ms 2024-02-19T21:51:46.911 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:45.485005+0000 mgr.smithi131.cjubsp (mgr.14164) 299 : cluster [DBG] pgmap v248: 97 pgs: 97 active+clean; 51 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.4 MiB/s wr, 102 op/s 2024-02-19T21:51:46.994 INFO:teuthology.orchestra.run.smithi131.stdout: 5 5916 39.10 MB/sec warmup 6 sec latency 13.043 ms 2024-02-19T21:51:47.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:45.485005+0000 mgr.smithi131.cjubsp (mgr.14164) 299 : cluster [DBG] pgmap v248: 97 pgs: 97 active+clean; 51 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.4 MiB/s wr, 102 op/s 2024-02-19T21:51:47.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 7055 40.29 MB/sec warmup 7 sec latency 18.420 ms 2024-02-19T21:51:48.634 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph config set global log_to_journald false --force' 2024-02-19T21:51:48.994 INFO:teuthology.orchestra.run.smithi131.stdout: 5 8011 37.46 MB/sec warmup 8 sec latency 20.326 ms 2024-02-19T21:51:49.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:47.485903+0000 mgr.smithi131.cjubsp (mgr.14164) 300 : cluster [DBG] pgmap v249: 97 pgs: 97 active+clean; 132 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 14 MiB/s wr, 784 op/s 2024-02-19T21:51:49.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:47.485903+0000 mgr.smithi131.cjubsp (mgr.14164) 300 : cluster [DBG] pgmap v249: 97 pgs: 97 active+clean; 132 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 14 MiB/s wr, 784 op/s 2024-02-19T21:51:49.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 9332 36.42 MB/sec warmup 9 sec latency 12.382 ms 2024-02-19T21:51:50.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 10639 37.81 MB/sec warmup 10 sec latency 14.022 ms 2024-02-19T21:51:51.144 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:49.486361+0000 mgr.smithi131.cjubsp (mgr.14164) 301 : cluster [DBG] pgmap v250: 97 pgs: 97 active+clean; 134 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 14 MiB/s wr, 813 op/s 2024-02-19T21:51:51.145 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:49.995207+0000 mon.smithi131 (mon.0) 680 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.145 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:50.447369+0000 mon.smithi131 (mon.0) 681 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.145 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:50.456992+0000 mon.smithi131 (mon.0) 682 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.145 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:50.459677+0000 mon.smithi131 (mon.0) 683 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:51.145 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:50.461148+0000 mon.smithi131 (mon.0) 684 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:51.145 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:50.462333+0000 mon.smithi131 (mon.0) 685 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:51.145 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:50.789027+0000 mon.smithi131 (mon.0) 686 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.146 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:50.798056+0000 mon.smithi131 (mon.0) 687 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:49.486361+0000 mgr.smithi131.cjubsp (mgr.14164) 301 : cluster [DBG] pgmap v250: 97 pgs: 97 active+clean; 134 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 14 MiB/s wr, 813 op/s 2024-02-19T21:51:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:49.995207+0000 mon.smithi131 (mon.0) 680 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:50.447369+0000 mon.smithi131 (mon.0) 681 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:50.456992+0000 mon.smithi131 (mon.0) 682 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:50.459677+0000 mon.smithi131 (mon.0) 683 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:50.461148+0000 mon.smithi131 (mon.0) 684 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:50.462333+0000 mon.smithi131 (mon.0) 685 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:50.789027+0000 mon.smithi131 (mon.0) 686 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:50.798056+0000 mon.smithi131 (mon.0) 687 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:51.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 11532 35.89 MB/sec warmup 11 sec latency 18.636 ms 2024-02-19T21:51:52.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 12933 35.32 MB/sec warmup 12 sec latency 53.253 ms 2024-02-19T21:51:53.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:51.486850+0000 mgr.smithi131.cjubsp (mgr.14164) 302 : cluster [DBG] pgmap v251: 97 pgs: 97 active+clean; 151 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 18 MiB/s wr, 1.26k op/s 2024-02-19T21:51:53.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:51.729567+0000 mon.smithi131 (mon.0) 688 : audit [INF] from='client.? ' entity='client.admin' 2024-02-19T21:51:53.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:51.735152+0000 mon.smithi131 (mon.0) 689 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:51.742104+0000 mon.smithi131 (mon.0) 690 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:51.743308+0000 mon.smithi131 (mon.0) 691 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:51.753783+0000 mon.smithi131 (mon.0) 692 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:51.761850+0000 mon.smithi131 (mon.0) 693 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:51.486850+0000 mgr.smithi131.cjubsp (mgr.14164) 302 : cluster [DBG] pgmap v251: 97 pgs: 97 active+clean; 151 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 18 MiB/s wr, 1.26k op/s 2024-02-19T21:51:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:51.729567+0000 mon.smithi131 (mon.0) 688 : audit [INF] from='client.? ' entity='client.admin' 2024-02-19T21:51:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:51.735152+0000 mon.smithi131 (mon.0) 689 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:51:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:51.742104+0000 mon.smithi131 (mon.0) 690 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:51:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:51.743308+0000 mon.smithi131 (mon.0) 691 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:51:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:51.753783+0000 mon.smithi131 (mon.0) 692 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:51.761850+0000 mon.smithi131 (mon.0) 693 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:51:53.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 14073 36.14 MB/sec warmup 13 sec latency 59.436 ms 2024-02-19T21:51:54.915 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph mgr module enable nfs --force' 2024-02-19T21:51:54.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 14928 34.90 MB/sec warmup 14 sec latency 59.872 ms 2024-02-19T21:51:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:53.487391+0000 mgr.smithi131.cjubsp (mgr.14164) 303 : cluster [DBG] pgmap v252: 97 pgs: 97 active+clean; 156 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 21 MiB/s wr, 1.48k op/s 2024-02-19T21:51:55.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:53.487391+0000 mgr.smithi131.cjubsp (mgr.14164) 303 : cluster [DBG] pgmap v252: 97 pgs: 97 active+clean; 156 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 21 MiB/s wr, 1.48k op/s 2024-02-19T21:51:55.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 15900 34.24 MB/sec warmup 15 sec latency 11.887 ms 2024-02-19T21:51:56.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 17366 34.62 MB/sec warmup 16 sec latency 13.719 ms 2024-02-19T21:51:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:55.487888+0000 mgr.smithi131.cjubsp (mgr.14164) 304 : cluster [DBG] pgmap v253: 97 pgs: 97 active+clean; 161 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 21 MiB/s wr, 1.61k op/s 2024-02-19T21:51:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:55.487888+0000 mgr.smithi131.cjubsp (mgr.14164) 304 : cluster [DBG] pgmap v253: 97 pgs: 97 active+clean; 161 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 21 MiB/s wr, 1.61k op/s 2024-02-19T21:51:57.822 INFO:teuthology.orchestra.run.smithi131.stderr:module 'nfs' is already enabled 2024-02-19T21:51:57.995 INFO:teuthology.orchestra.run.smithi131.stdout: 5 18458 34.36 MB/sec warmup 17 sec latency 17.070 ms 2024-02-19T21:51:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:57.402079+0000 mon.smithi196 (mon.1) 16 : audit [INF] from='client.? 172.21.15.131:0/2757885289' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2024-02-19T21:51:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:57.402605+0000 mon.smithi131 (mon.0) 694 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2024-02-19T21:51:58.202 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:57.402079+0000 mon.smithi196 (mon.1) 16 : audit [INF] from='client.? 172.21.15.131:0/2757885289' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2024-02-19T21:51:58.202 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:57.402605+0000 mon.smithi131 (mon.0) 694 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2024-02-19T21:51:58.664 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-02-19T21:51:58.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 19562 33.87 MB/sec warmup 18 sec latency 20.744 ms 2024-02-19T21:51:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:57.488481+0000 mgr.smithi131.cjubsp (mgr.14164) 305 : cluster [DBG] pgmap v254: 97 pgs: 97 active+clean; 168 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 23 MiB/s wr, 1.92k op/s 2024-02-19T21:51:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:51:57.808281+0000 mon.smithi131 (mon.0) 695 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]': finished 2024-02-19T21:51:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:51:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:57.808354+0000 mon.smithi131 (mon.0) 696 : cluster [DBG] mgrmap e20: smithi131.cjubsp(active, since 6m), standbys: smithi196.voprex 2024-02-19T21:51:59.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:57.488481+0000 mgr.smithi131.cjubsp (mgr.14164) 305 : cluster [DBG] pgmap v254: 97 pgs: 97 active+clean; 168 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 23 MiB/s wr, 1.92k op/s 2024-02-19T21:51:59.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:51:57.808281+0000 mon.smithi131 (mon.0) 695 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]': finished 2024-02-19T21:51:59.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:51:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:57.808354+0000 mon.smithi131 (mon.0) 696 : cluster [DBG] mgrmap e20: smithi131.cjubsp(active, since 6m), standbys: smithi196.voprex 2024-02-19T21:51:59.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 20772 33.96 MB/sec warmup 19 sec latency 20.304 ms 2024-02-19T21:52:00.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 21796 33.93 MB/sec warmup 20 sec latency 14.337 ms 2024-02-19T21:52:01.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:51:59.488943+0000 mgr.smithi131.cjubsp (mgr.14164) 306 : cluster [DBG] pgmap v255: 97 pgs: 97 active+clean; 167 MiB data, 931 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 14 MiB/s wr, 1.27k op/s 2024-02-19T21:52:01.167 INFO:teuthology.orchestra.run.smithi131.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1 2024-02-19T21:52:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:51:59.488943+0000 mgr.smithi131.cjubsp (mgr.14164) 306 : cluster [DBG] pgmap v255: 97 pgs: 97 active+clean; 167 MiB data, 931 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 14 MiB/s wr, 1.27k op/s 2024-02-19T21:52:01.991 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-02-19T21:52:01.992 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T21:52:01.992 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-02-19T21:52:01.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 22835 33.53 MB/sec warmup 21 sec latency 15.792 ms 2024-02-19T21:52:02.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:01.155851+0000 mgr.smithi131.cjubsp (mgr.14164) 307 : audit [DBG] from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:02.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:52:01.156649+0000 mgr.smithi131.cjubsp (mgr.14164) 308 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1 2024-02-19T21:52:02.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:01.167044+0000 mon.smithi131 (mon.0) 697 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:01.167613+0000 mon.smithi131 (mon.0) 698 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:52:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:01.168704+0000 mon.smithi131 (mon.0) 699 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:52:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:01.169413+0000 mon.smithi131 (mon.0) 700 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:52:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:01.174486+0000 mon.smithi131 (mon.0) 701 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:01.181764+0000 mon.smithi131 (mon.0) 702 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:52:01.183580+0000 mgr.smithi131.cjubsp (mgr.14164) 309 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1 2024-02-19T21:52:02.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:01.489426+0000 mgr.smithi131.cjubsp (mgr.14164) 310 : cluster [DBG] pgmap v256: 97 pgs: 97 active+clean; 179 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 17 MiB/s wr, 1.60k op/s 2024-02-19T21:52:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:01.155851+0000 mgr.smithi131.cjubsp (mgr.14164) 307 : audit [DBG] from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:52:01.156649+0000 mgr.smithi131.cjubsp (mgr.14164) 308 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1 2024-02-19T21:52:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:01.167044+0000 mon.smithi131 (mon.0) 697 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:01.167613+0000 mon.smithi131 (mon.0) 698 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:52:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:01.168704+0000 mon.smithi131 (mon.0) 699 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:52:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:01.169413+0000 mon.smithi131 (mon.0) 700 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:52:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:01.174486+0000 mon.smithi131 (mon.0) 701 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:01.181764+0000 mon.smithi131 (mon.0) 702 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:52:01.183580+0000 mgr.smithi131.cjubsp (mgr.14164) 309 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1 2024-02-19T21:52:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:01.489426+0000 mgr.smithi131.cjubsp (mgr.14164) 310 : cluster [DBG] pgmap v256: 97 pgs: 97 active+clean; 179 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 17 MiB/s wr, 1.60k op/s 2024-02-19T21:52:02.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 23870 32.73 MB/sec warmup 22 sec latency 15.315 ms 2024-02-19T21:52:03.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 25152 33.61 MB/sec warmup 23 sec latency 26.467 ms 2024-02-19T21:52:04.820 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:52:04.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 25825 32.47 MB/sec warmup 24 sec latency 26.910 ms 2024-02-19T21:52:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:03.489948+0000 mgr.smithi131.cjubsp (mgr.14164) 311 : cluster [DBG] pgmap v257: 97 pgs: 97 active+clean; 183 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 16 MiB/s wr, 1.34k op/s 2024-02-19T21:52:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.729746+0000 mon.smithi131 (mon.0) 703 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-02-19T21:52:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.729991+0000 mon.smithi131 (mon.0) 704 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-02-19T21:52:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.730131+0000 mon.smithi131 (mon.0) 705 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-02-19T21:52:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.730246+0000 mon.smithi131 (mon.0) 706 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-02-19T21:52:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:03.489948+0000 mgr.smithi131.cjubsp (mgr.14164) 311 : cluster [DBG] pgmap v257: 97 pgs: 97 active+clean; 183 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 16 MiB/s wr, 1.34k op/s 2024-02-19T21:52:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.729746+0000 mon.smithi131 (mon.0) 703 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-02-19T21:52:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.729991+0000 mon.smithi131 (mon.0) 704 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-02-19T21:52:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.730131+0000 mon.smithi131 (mon.0) 705 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-02-19T21:52:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.730246+0000 mon.smithi131 (mon.0) 706 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-02-19T21:52:05.727 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:52:05.727 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (4m) 15s ago 6m 21.3M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:52:05.727 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (6m) 15s ago 6m 6387k - 16.2.5 6933c2a0b7dd 7e4833221591 2024-02-19T21:52:05.727 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (5m) 24s ago 5m 6395k - 16.2.5 6933c2a0b7dd d566245c5426 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (4m) 15s ago 6m 34.1M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (47s) 15s ago 47s 13.6M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (45s) 24s ago 44s 11.8M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:9283 running (8m) 15s ago 7m 422M - 16.2.5 6933c2a0b7dd 785c7b05f4ea 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (5m) 24s ago 5m 371M - 16.2.5 6933c2a0b7dd 6776b8183cbb 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (8m) 15s ago 8m 44.8M 2048M 16.2.5 6933c2a0b7dd d880a31c6170 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (5m) 24s ago 5m 38.1M 2048M 16.2.5 6933c2a0b7dd 6a272417d66d 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.0.smithi131.nauhyt smithi131 *:2049 running (33s) 15s ago 32s 53.6M - 3.5 6933c2a0b7dd fc874552aefb 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.0.smithi196.hmjmsf smithi196 *:2049 running (27s) 24s ago 27s 53.5M - 3.5 6933c2a0b7dd af920ee7c215 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (6m) 15s ago 6m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (5m) 24s ago 5m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (4m) 15s ago 4m 36.5M 3224M 16.2.5 6933c2a0b7dd f42dfded3a0d 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (4m) 15s ago 4m 38.0M 3224M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (4m) 15s ago 4m 35.1M 3224M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:52:05.728 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (3m) 15s ago 3m 39.1M 3224M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:52:05.729 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (3m) 24s ago 3m 35.0M 3992M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:52:05.729 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (3m) 24s ago 3m 36.8M 3992M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:52:05.729 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (2m) 24s ago 2m 37.2M 3992M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:52:05.729 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (2m) 24s ago 2m 39.5M 3992M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:52:05.729 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (4m) 15s ago 5m 40.6M - 2.18.1 de242295e225 22b1466d4ddf 2024-02-19T21:52:05.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 26871 32.08 MB/sec warmup 25 sec latency 23.322 ms 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.800202+0000 mgr.smithi131.cjubsp (mgr.14164) 312 : audit [DBG] from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.816348+0000 mon.smithi131 (mon.0) 707 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.816438+0000 mon.smithi131 (mon.0) 708 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.816512+0000 mon.smithi131 (mon.0) 709 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.816543+0000 mon.smithi131 (mon.0) 710 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:04.816567+0000 mon.smithi131 (mon.0) 711 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.865122+0000 mon.smithi131 (mon.0) 712 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:04.865316+0000 mon.smithi131 (mon.0) 713 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:52:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:05.821330+0000 mon.smithi131 (mon.0) 714 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.800202+0000 mgr.smithi131.cjubsp (mgr.14164) 312 : audit [DBG] from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.816348+0000 mon.smithi131 (mon.0) 707 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.816438+0000 mon.smithi131 (mon.0) 708 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.816512+0000 mon.smithi131 (mon.0) 709 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.816543+0000 mon.smithi131 (mon.0) 710 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:04.816567+0000 mon.smithi131 (mon.0) 711 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.865122+0000 mon.smithi131 (mon.0) 712 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:04.865316+0000 mon.smithi131 (mon.0) 713 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:52:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:05.821330+0000 mon.smithi131 (mon.0) 714 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-02-19T21:52:06.298 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:52:06.298 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:52:06.298 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:52:06.298 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:52:06.298 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:52:06.298 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:52:06.298 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:52:06.298 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 14 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:52:06.299 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:52:06.791 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:52:06.791 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1", 2024-02-19T21:52:06.791 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:52:06.791 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [], 2024-02-19T21:52:06.791 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "", 2024-02-19T21:52:06.791 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:81bd20d634209c7cb82c18be12b4b5a05643ebf1 image" 2024-02-19T21:52:06.791 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:52:06.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 27843 32.13 MB/sec warmup 26 sec latency 19.482 ms 2024-02-19T21:52:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:05.253912+0000 mgr.smithi131.cjubsp (mgr.14164) 313 : audit [DBG] from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:05.490403+0000 mgr.smithi131.cjubsp (mgr.14164) 314 : cluster [DBG] pgmap v259: 97 pgs: 1 peering, 96 active+clean; 193 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 16 MiB/s wr, 1.44k op/s 2024-02-19T21:52:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:05.712151+0000 mgr.smithi131.cjubsp (mgr.14164) 315 : audit [DBG] from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:06.299296+0000 mon.smithi196 (mon.1) 17 : audit [DBG] from='client.? 172.21.15.131:0/1702216057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:52:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:06.819844+0000 mon.smithi131 (mon.0) 715 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-02-19T21:52:07.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:05.253912+0000 mgr.smithi131.cjubsp (mgr.14164) 313 : audit [DBG] from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:07.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:05.490403+0000 mgr.smithi131.cjubsp (mgr.14164) 314 : cluster [DBG] pgmap v259: 97 pgs: 1 peering, 96 active+clean; 193 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 16 MiB/s wr, 1.44k op/s 2024-02-19T21:52:07.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:05.712151+0000 mgr.smithi131.cjubsp (mgr.14164) 315 : audit [DBG] from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:07.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:06.299296+0000 mon.smithi196 (mon.1) 17 : audit [DBG] from='client.? 172.21.15.131:0/1702216057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:52:07.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:06.819844+0000 mon.smithi131 (mon.0) 715 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-02-19T21:52:07.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 28734 32.17 MB/sec warmup 27 sec latency 25.600 ms 2024-02-19T21:52:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:06.791988+0000 mgr.smithi131.cjubsp (mgr.14164) 316 : audit [DBG] from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:08.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:06.791988+0000 mgr.smithi131.cjubsp (mgr.14164) 316 : audit [DBG] from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:08.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 29609 31.54 MB/sec warmup 28 sec latency 18.218 ms 2024-02-19T21:52:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:07.490867+0000 mgr.smithi131.cjubsp (mgr.14164) 317 : cluster [DBG] pgmap v262: 97 pgs: 1 remapped+peering, 1 peering, 95 active+clean; 202 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 17 MiB/s wr, 1.56k op/s 2024-02-19T21:52:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:07.861577+0000 mon.smithi131 (mon.0) 716 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-02-19T21:52:09.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:07.490867+0000 mgr.smithi131.cjubsp (mgr.14164) 317 : cluster [DBG] pgmap v262: 97 pgs: 1 remapped+peering, 1 peering, 95 active+clean; 202 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 17 MiB/s wr, 1.56k op/s 2024-02-19T21:52:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:07.861577+0000 mon.smithi131 (mon.0) 716 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-02-19T21:52:09.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 30825 31.23 MB/sec warmup 29 sec latency 17.005 ms 2024-02-19T21:52:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:08.869822+0000 mon.smithi131 (mon.0) 717 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-02-19T21:52:10.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:08.869822+0000 mon.smithi131 (mon.0) 717 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-02-19T21:52:10.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 31753 31.45 MB/sec warmup 30 sec latency 19.387 ms 2024-02-19T21:52:11.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:09.491277+0000 mgr.smithi131.cjubsp (mgr.14164) 318 : cluster [DBG] pgmap v265: 97 pgs: 1 remapped+peering, 1 peering, 95 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 16 MiB/s wr, 1.40k op/s 2024-02-19T21:52:11.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:09.491277+0000 mgr.smithi131.cjubsp (mgr.14164) 318 : cluster [DBG] pgmap v265: 97 pgs: 1 remapped+peering, 1 peering, 95 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 16 MiB/s wr, 1.40k op/s 2024-02-19T21:52:11.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 32339 31.13 MB/sec warmup 31 sec latency 24.274 ms 2024-02-19T21:52:12.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 33094 30.48 MB/sec warmup 32 sec latency 24.567 ms 2024-02-19T21:52:13.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:11.500231+0000 mgr.smithi131.cjubsp (mgr.14164) 319 : cluster [DBG] pgmap v266: 97 pgs: 1 remapped+peering, 96 active+clean; 214 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 16 MiB/s wr, 1.53k op/s 2024-02-19T21:52:13.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:11.500231+0000 mgr.smithi131.cjubsp (mgr.14164) 319 : cluster [DBG] pgmap v266: 97 pgs: 1 remapped+peering, 96 active+clean; 214 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 16 MiB/s wr, 1.53k op/s 2024-02-19T21:52:13.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 34277 30.24 MB/sec warmup 33 sec latency 18.149 ms 2024-02-19T21:52:14.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 35495 30.81 MB/sec warmup 34 sec latency 16.689 ms 2024-02-19T21:52:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:13.500753+0000 mgr.smithi131.cjubsp (mgr.14164) 320 : cluster [DBG] pgmap v267: 97 pgs: 97 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.13k op/s; 613 KiB/s, 0 objects/s recovering 2024-02-19T21:52:15.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:14.891750+0000 mon.smithi131 (mon.0) 718 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:15.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:13.500753+0000 mgr.smithi131.cjubsp (mgr.14164) 320 : cluster [DBG] pgmap v267: 97 pgs: 97 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.13k op/s; 613 KiB/s, 0 objects/s recovering 2024-02-19T21:52:15.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:14.891750+0000 mon.smithi131 (mon.0) 718 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:15.997 INFO:teuthology.orchestra.run.smithi131.stdout: 5 36415 30.44 MB/sec warmup 35 sec latency 24.303 ms 2024-02-19T21:52:16.997 INFO:teuthology.orchestra.run.smithi131.stdout: 5 37359 30.23 MB/sec warmup 36 sec latency 18.731 ms 2024-02-19T21:52:17.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:15.501232+0000 mgr.smithi131.cjubsp (mgr.14164) 321 : cluster [DBG] pgmap v268: 97 pgs: 97 active+clean; 220 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 11 MiB/s wr, 1.08k op/s; 511 KiB/s, 0 objects/s recovering 2024-02-19T21:52:17.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:15.501232+0000 mgr.smithi131.cjubsp (mgr.14164) 321 : cluster [DBG] pgmap v268: 97 pgs: 97 active+clean; 220 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 11 MiB/s wr, 1.08k op/s; 511 KiB/s, 0 objects/s recovering 2024-02-19T21:52:17.997 INFO:teuthology.orchestra.run.smithi131.stdout: 5 38156 29.95 MB/sec warmup 37 sec latency 43.135 ms 2024-02-19T21:52:18.996 INFO:teuthology.orchestra.run.smithi131.stdout: 5 39226 30.31 MB/sec warmup 38 sec latency 21.791 ms 2024-02-19T21:52:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:17.501854+0000 mgr.smithi131.cjubsp (mgr.14164) 322 : cluster [DBG] pgmap v269: 97 pgs: 97 active+clean; 234 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 14 MiB/s wr, 1.36k op/s; 425 KiB/s, 0 objects/s recovering 2024-02-19T21:52:19.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:17.501854+0000 mgr.smithi131.cjubsp (mgr.14164) 322 : cluster [DBG] pgmap v269: 97 pgs: 97 active+clean; 234 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 14 MiB/s wr, 1.36k op/s; 425 KiB/s, 0 objects/s recovering 2024-02-19T21:52:19.997 INFO:teuthology.orchestra.run.smithi131.stdout: 5 39898 29.73 MB/sec warmup 39 sec latency 21.938 ms 2024-02-19T21:52:20.997 INFO:teuthology.orchestra.run.smithi131.stdout: 5 40869 29.59 MB/sec warmup 40 sec latency 24.490 ms 2024-02-19T21:52:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:19.502334+0000 mgr.smithi131.cjubsp (mgr.14164) 323 : cluster [DBG] pgmap v270: 97 pgs: 97 active+clean; 232 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 13 MiB/s wr, 1.12k op/s; 385 KiB/s, 0 objects/s recovering 2024-02-19T21:52:21.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:19.502334+0000 mgr.smithi131.cjubsp (mgr.14164) 323 : cluster [DBG] pgmap v270: 97 pgs: 97 active+clean; 232 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 13 MiB/s wr, 1.12k op/s; 385 KiB/s, 0 objects/s recovering 2024-02-19T21:52:21.997 INFO:teuthology.orchestra.run.smithi131.stdout: 5 41694 29.34 MB/sec warmup 41 sec latency 27.977 ms 2024-02-19T21:52:22.997 INFO:teuthology.orchestra.run.smithi131.stdout: 5 42566 29.45 MB/sec warmup 42 sec latency 18.175 ms 2024-02-19T21:52:23.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:21.502839+0000 mgr.smithi131.cjubsp (mgr.14164) 324 : cluster [DBG] pgmap v271: 97 pgs: 97 active+clean; 243 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.26k op/s; 341 KiB/s, 0 objects/s recovering 2024-02-19T21:52:23.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:21.502839+0000 mgr.smithi131.cjubsp (mgr.14164) 324 : cluster [DBG] pgmap v271: 97 pgs: 97 active+clean; 243 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.26k op/s; 341 KiB/s, 0 objects/s recovering 2024-02-19T21:52:23.997 INFO:teuthology.orchestra.run.smithi131.stdout: 5 43223 29.15 MB/sec warmup 43 sec latency 18.935 ms 2024-02-19T21:52:24.998 INFO:teuthology.orchestra.run.smithi131.stdout: 5 44181 29.05 MB/sec warmup 44 sec latency 15.147 ms 2024-02-19T21:52:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:23.503363+0000 mgr.smithi131.cjubsp (mgr.14164) 325 : cluster [DBG] pgmap v272: 97 pgs: 97 active+clean; 243 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.18k op/s; 341 KiB/s, 0 objects/s recovering 2024-02-19T21:52:25.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:23.503363+0000 mgr.smithi131.cjubsp (mgr.14164) 325 : cluster [DBG] pgmap v272: 97 pgs: 97 active+clean; 243 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.18k op/s; 341 KiB/s, 0 objects/s recovering 2024-02-19T21:52:25.998 INFO:teuthology.orchestra.run.smithi131.stdout: 5 45213 28.82 MB/sec warmup 45 sec latency 32.478 ms 2024-02-19T21:52:26.998 INFO:teuthology.orchestra.run.smithi131.stdout: 5 46232 29.17 MB/sec warmup 46 sec latency 20.809 ms 2024-02-19T21:52:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:25.504388+0000 mgr.smithi131.cjubsp (mgr.14164) 326 : cluster [DBG] pgmap v273: 97 pgs: 97 active+clean; 253 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-02-19T21:52:27.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:25.504388+0000 mgr.smithi131.cjubsp (mgr.14164) 326 : cluster [DBG] pgmap v273: 97 pgs: 97 active+clean; 253 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-02-19T21:52:27.998 INFO:teuthology.orchestra.run.smithi131.stdout: 5 46861 28.71 MB/sec warmup 47 sec latency 19.253 ms 2024-02-19T21:52:28.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 47521 28.43 MB/sec warmup 48 sec latency 31.860 ms 2024-02-19T21:52:29.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:27.504895+0000 mgr.smithi131.cjubsp (mgr.14164) 327 : cluster [DBG] pgmap v274: 97 pgs: 97 active+clean; 259 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 15 MiB/s wr, 1.36k op/s 2024-02-19T21:52:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:27.504895+0000 mgr.smithi131.cjubsp (mgr.14164) 327 : cluster [DBG] pgmap v274: 97 pgs: 97 active+clean; 259 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 15 MiB/s wr, 1.36k op/s 2024-02-19T21:52:29.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 48605 28.28 MB/sec warmup 49 sec latency 24.513 ms 2024-02-19T21:52:30.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 49691 28.57 MB/sec warmup 50 sec latency 16.955 ms 2024-02-19T21:52:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:29.505335+0000 mgr.smithi131.cjubsp (mgr.14164) 328 : cluster [DBG] pgmap v275: 97 pgs: 97 active+clean; 259 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-02-19T21:52:31.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:29.505335+0000 mgr.smithi131.cjubsp (mgr.14164) 328 : cluster [DBG] pgmap v275: 97 pgs: 97 active+clean; 259 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-02-19T21:52:31.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 50616 28.40 MB/sec warmup 51 sec latency 15.091 ms 2024-02-19T21:52:32.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 51811 28.36 MB/sec warmup 52 sec latency 25.137 ms 2024-02-19T21:52:33.254 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:31.505852+0000 mgr.smithi131.cjubsp (mgr.14164) 329 : cluster [DBG] pgmap v276: 97 pgs: 97 active+clean; 264 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.22k op/s 2024-02-19T21:52:33.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:31.505852+0000 mgr.smithi131.cjubsp (mgr.14164) 329 : cluster [DBG] pgmap v276: 97 pgs: 97 active+clean; 264 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.22k op/s 2024-02-19T21:52:33.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 52842 28.56 MB/sec warmup 53 sec latency 22.326 ms 2024-02-19T21:52:34.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 53604 28.47 MB/sec warmup 54 sec latency 23.441 ms 2024-02-19T21:52:35.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:33.506408+0000 mgr.smithi131.cjubsp (mgr.14164) 330 : cluster [DBG] pgmap v277: 97 pgs: 97 active+clean; 275 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.19k op/s 2024-02-19T21:52:35.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:34.064853+0000 mon.smithi131 (mon.0) 719 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:35.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:34.066271+0000 mon.smithi131 (mon.0) 720 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:52:35.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:33.506408+0000 mgr.smithi131.cjubsp (mgr.14164) 330 : cluster [DBG] pgmap v277: 97 pgs: 97 active+clean; 275 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.19k op/s 2024-02-19T21:52:35.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:34.064853+0000 mon.smithi131 (mon.0) 719 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:52:35.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:34.066271+0000 mon.smithi131 (mon.0) 720 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:52:35.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 54489 28.24 MB/sec warmup 55 sec latency 23.999 ms 2024-02-19T21:52:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:52:34.065377+0000 mgr.smithi131.cjubsp (mgr.14164) 331 : cephadm [INF] Upgrade: Target is version 16.2.14-548-g81bd20d6 (pacific) 2024-02-19T21:52:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:52:34.065460+0000 mgr.smithi131.cjubsp (mgr.14164) 332 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9'] 2024-02-19T21:52:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:52:34.067021+0000 mgr.smithi131.cjubsp (mgr.14164) 333 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi131.cjubsp) 2024-02-19T21:52:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:52:34.377349+0000 mgr.smithi131.cjubsp (mgr.14164) 334 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9 on smithi196 2024-02-19T21:52:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:52:34.065377+0000 mgr.smithi131.cjubsp (mgr.14164) 331 : cephadm [INF] Upgrade: Target is version 16.2.14-548-g81bd20d6 (pacific) 2024-02-19T21:52:36.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:52:34.065460+0000 mgr.smithi131.cjubsp (mgr.14164) 332 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9'] 2024-02-19T21:52:36.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:52:34.067021+0000 mgr.smithi131.cjubsp (mgr.14164) 333 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi131.cjubsp) 2024-02-19T21:52:36.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:52:34.377349+0000 mgr.smithi131.cjubsp (mgr.14164) 334 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9 on smithi196 2024-02-19T21:52:36.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 55246 28.02 MB/sec warmup 56 sec latency 42.127 ms 2024-02-19T21:52:37.183 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:52:37.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:35.506924+0000 mgr.smithi131.cjubsp (mgr.14164) 335 : cluster [DBG] pgmap v278: 97 pgs: 97 active+clean; 282 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.21k op/s 2024-02-19T21:52:37.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:35.506924+0000 mgr.smithi131.cjubsp (mgr.14164) 335 : cluster [DBG] pgmap v278: 97 pgs: 97 active+clean; 282 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.21k op/s 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (5m) 47s ago 7m 21.3M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (7m) 47s ago 7m 6387k - 16.2.5 6933c2a0b7dd 7e4833221591 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (6m) 56s ago 6m 6395k - 16.2.5 6933c2a0b7dd d566245c5426 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (5m) 47s ago 6m 34.1M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (79s) 47s ago 79s 13.6M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (77s) 56s ago 76s 11.8M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:9283 running (8m) 47s ago 8m 422M - 16.2.5 6933c2a0b7dd 785c7b05f4ea 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (6m) 56s ago 6m 371M - 16.2.5 6933c2a0b7dd 6776b8183cbb 2024-02-19T21:52:37.872 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (8m) 47s ago 8m 44.8M 2048M 16.2.5 6933c2a0b7dd d880a31c6170 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (6m) 56s ago 6m 38.1M 2048M 16.2.5 6933c2a0b7dd 6a272417d66d 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.0.smithi131.nauhyt smithi131 *:2049 running (65s) 47s ago 64s 53.6M - 3.5 6933c2a0b7dd fc874552aefb 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.0.smithi196.hmjmsf smithi196 *:2049 running (59s) 56s ago 59s 53.5M - 3.5 6933c2a0b7dd af920ee7c215 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (6m) 47s ago 6m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (5m) 56s ago 5m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (5m) 47s ago 5m 36.5M 3224M 16.2.5 6933c2a0b7dd f42dfded3a0d 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (5m) 47s ago 5m 38.0M 3224M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (4m) 47s ago 4m 35.1M 3224M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (4m) 47s ago 4m 39.1M 3224M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (4m) 56s ago 3m 35.0M 3992M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (3m) 56s ago 3m 36.8M 3992M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (3m) 56s ago 3m 37.2M 3992M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (2m) 56s ago 2m 39.5M 3992M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:52:37.873 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (5m) 47s ago 6m 40.6M - 2.18.1 de242295e225 22b1466d4ddf 2024-02-19T21:52:37.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 55939 27.84 MB/sec warmup 57 sec latency 21.120 ms 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2024-02-19T21:52:38.432 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:52:38.433 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:52:38.433 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:52:38.433 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:52:38.433 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:52:38.433 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 14 2024-02-19T21:52:38.433 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:52:38.433 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:52:38.893 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:52:38.894 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:52:38.894 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:52:38.894 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [], 2024-02-19T21:52:38.894 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "0/23 ceph daemons upgraded", 2024-02-19T21:52:38.894 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9 image on host smithi196" 2024-02-19T21:52:38.894 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:52:38.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 56842 27.99 MB/sec warmup 58 sec latency 23.702 ms 2024-02-19T21:52:39.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:37.168042+0000 mgr.smithi131.cjubsp (mgr.14164) 336 : audit [DBG] from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:39.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:37.507407+0000 mgr.smithi131.cjubsp (mgr.14164) 337 : cluster [DBG] pgmap v279: 97 pgs: 97 active+clean; 284 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 14 MiB/s wr, 1.26k op/s 2024-02-19T21:52:39.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:37.523671+0000 mgr.smithi131.cjubsp (mgr.14164) 338 : audit [DBG] from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:39.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:37.859296+0000 mgr.smithi131.cjubsp (mgr.14164) 339 : audit [DBG] from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:39.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:38.432659+0000 mon.smithi131 (mon.0) 721 : audit [DBG] from='client.? 172.21.15.131:0/99638103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:52:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:37.168042+0000 mgr.smithi131.cjubsp (mgr.14164) 336 : audit [DBG] from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:37.507407+0000 mgr.smithi131.cjubsp (mgr.14164) 337 : cluster [DBG] pgmap v279: 97 pgs: 97 active+clean; 284 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 14 MiB/s wr, 1.26k op/s 2024-02-19T21:52:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:37.523671+0000 mgr.smithi131.cjubsp (mgr.14164) 338 : audit [DBG] from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:37.859296+0000 mgr.smithi131.cjubsp (mgr.14164) 339 : audit [DBG] from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:38.432659+0000 mon.smithi131 (mon.0) 721 : audit [DBG] from='client.? 172.21.15.131:0/99638103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:52:39.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 57671 27.79 MB/sec warmup 59 sec latency 14.707 ms 2024-02-19T21:52:40.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:52:38.894201+0000 mgr.smithi131.cjubsp (mgr.14164) 340 : audit [DBG] from='client.24423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:52:38.894201+0000 mgr.smithi131.cjubsp (mgr.14164) 340 : audit [DBG] from='client.24423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:52:40.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 58498 27.61 MB/sec warmup 60 sec latency 32.581 ms 2024-02-19T21:52:41.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:39.507779+0000 mgr.smithi131.cjubsp (mgr.14164) 341 : cluster [DBG] pgmap v280: 97 pgs: 97 active+clean; 288 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-02-19T21:52:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:39.507779+0000 mgr.smithi131.cjubsp (mgr.14164) 341 : cluster [DBG] pgmap v280: 97 pgs: 97 active+clean; 288 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-02-19T21:52:41.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 59125 27.35 MB/sec warmup 61 sec latency 37.230 ms 2024-02-19T21:52:42.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 59537 27.15 MB/sec warmup 62 sec latency 39.948 ms 2024-02-19T21:52:43.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:41.508183+0000 mgr.smithi131.cjubsp (mgr.14164) 342 : cluster [DBG] pgmap v281: 97 pgs: 97 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.22k op/s 2024-02-19T21:52:43.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:41.508183+0000 mgr.smithi131.cjubsp (mgr.14164) 342 : cluster [DBG] pgmap v281: 97 pgs: 97 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.22k op/s 2024-02-19T21:52:43.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 60164 27.17 MB/sec warmup 63 sec latency 35.858 ms 2024-02-19T21:52:44.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 60898 27.05 MB/sec warmup 64 sec latency 20.651 ms 2024-02-19T21:52:45.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:43.508641+0000 mgr.smithi131.cjubsp (mgr.14164) 343 : cluster [DBG] pgmap v282: 97 pgs: 97 active+clean; 301 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-02-19T21:52:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:43.508641+0000 mgr.smithi131.cjubsp (mgr.14164) 343 : cluster [DBG] pgmap v282: 97 pgs: 97 active+clean; 301 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-02-19T21:52:45.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 61574 26.83 MB/sec warmup 65 sec latency 25.538 ms 2024-02-19T21:52:47.000 INFO:teuthology.orchestra.run.smithi131.stdout: 5 62464 26.70 MB/sec warmup 66 sec latency 25.346 ms 2024-02-19T21:52:47.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:45.509060+0000 mgr.smithi131.cjubsp (mgr.14164) 344 : cluster [DBG] pgmap v283: 97 pgs: 97 active+clean; 301 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 1.10k op/s 2024-02-19T21:52:47.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:45.509060+0000 mgr.smithi131.cjubsp (mgr.14164) 344 : cluster [DBG] pgmap v283: 97 pgs: 97 active+clean; 301 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 1.10k op/s 2024-02-19T21:52:48.000 INFO:teuthology.orchestra.run.smithi131.stdout: 5 63524 26.87 MB/sec warmup 67 sec latency 13.317 ms 2024-02-19T21:52:48.999 INFO:teuthology.orchestra.run.smithi131.stdout: 5 64238 26.81 MB/sec warmup 68 sec latency 99.075 ms 2024-02-19T21:52:49.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:47.509548+0000 mgr.smithi131.cjubsp (mgr.14164) 345 : cluster [DBG] pgmap v284: 97 pgs: 97 active+clean; 307 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-02-19T21:52:49.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:47.509548+0000 mgr.smithi131.cjubsp (mgr.14164) 345 : cluster [DBG] pgmap v284: 97 pgs: 97 active+clean; 307 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-02-19T21:52:50.000 INFO:teuthology.orchestra.run.smithi131.stdout: 5 64865 26.58 MB/sec warmup 69 sec latency 22.379 ms 2024-02-19T21:52:51.000 INFO:teuthology.orchestra.run.smithi131.stdout: 5 65577 26.40 MB/sec warmup 70 sec latency 38.269 ms 2024-02-19T21:52:51.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:49.509979+0000 mgr.smithi131.cjubsp (mgr.14164) 346 : cluster [DBG] pgmap v285: 97 pgs: 97 active+clean; 308 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 9.7 MiB/s wr, 952 op/s 2024-02-19T21:52:51.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:49.509979+0000 mgr.smithi131.cjubsp (mgr.14164) 346 : cluster [DBG] pgmap v285: 97 pgs: 97 active+clean; 308 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 9.7 MiB/s wr, 952 op/s 2024-02-19T21:52:52.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 66399 26.32 MB/sec warmup 71 sec latency 21.273 ms 2024-02-19T21:52:53.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 67190 26.41 MB/sec warmup 72 sec latency 20.055 ms 2024-02-19T21:52:53.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:51.510525+0000 mgr.smithi131.cjubsp (mgr.14164) 347 : cluster [DBG] pgmap v286: 97 pgs: 97 active+clean; 313 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-02-19T21:52:53.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:51.510525+0000 mgr.smithi131.cjubsp (mgr.14164) 347 : cluster [DBG] pgmap v286: 97 pgs: 97 active+clean; 313 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-02-19T21:52:54.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 67831 26.29 MB/sec warmup 73 sec latency 48.267 ms 2024-02-19T21:52:55.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 68239 26.03 MB/sec warmup 74 sec latency 28.013 ms 2024-02-19T21:52:55.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:53.511070+0000 mgr.smithi131.cjubsp (mgr.14164) 348 : cluster [DBG] pgmap v287: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.11k op/s 2024-02-19T21:52:55.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:53.511070+0000 mgr.smithi131.cjubsp (mgr.14164) 348 : cluster [DBG] pgmap v287: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.11k op/s 2024-02-19T21:52:56.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 69060 25.91 MB/sec warmup 75 sec latency 21.332 ms 2024-02-19T21:52:57.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 69860 25.84 MB/sec warmup 76 sec latency 25.203 ms 2024-02-19T21:52:57.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:55.511557+0000 mgr.smithi131.cjubsp (mgr.14164) 349 : cluster [DBG] pgmap v288: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.8 MiB/s wr, 1.05k op/s 2024-02-19T21:52:57.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:55.511557+0000 mgr.smithi131.cjubsp (mgr.14164) 349 : cluster [DBG] pgmap v288: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.8 MiB/s wr, 1.05k op/s 2024-02-19T21:52:58.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 70699 25.91 MB/sec warmup 77 sec latency 18.486 ms 2024-02-19T21:52:59.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 71277 25.82 MB/sec warmup 78 sec latency 41.689 ms 2024-02-19T21:52:59.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:52:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:57.512105+0000 mgr.smithi131.cjubsp (mgr.14164) 350 : cluster [DBG] pgmap v289: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.22k op/s 2024-02-19T21:52:59.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:52:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:57.512105+0000 mgr.smithi131.cjubsp (mgr.14164) 350 : cluster [DBG] pgmap v289: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.22k op/s 2024-02-19T21:53:00.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 71574 25.55 MB/sec warmup 79 sec latency 33.538 ms 2024-02-19T21:53:01.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 71900 25.31 MB/sec warmup 80 sec latency 60.480 ms 2024-02-19T21:53:01.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:52:59.512497+0000 mgr.smithi131.cjubsp (mgr.14164) 351 : cluster [DBG] pgmap v290: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 941 op/s 2024-02-19T21:53:01.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:52:59.512497+0000 mgr.smithi131.cjubsp (mgr.14164) 351 : cluster [DBG] pgmap v290: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 941 op/s 2024-02-19T21:53:02.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 72621 25.17 MB/sec warmup 81 sec latency 58.599 ms 2024-02-19T21:53:03.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 73347 25.10 MB/sec warmup 82 sec latency 32.928 ms 2024-02-19T21:53:03.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:01.512892+0000 mgr.smithi131.cjubsp (mgr.14164) 352 : cluster [DBG] pgmap v291: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-02-19T21:53:03.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:01.512892+0000 mgr.smithi131.cjubsp (mgr.14164) 352 : cluster [DBG] pgmap v291: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-02-19T21:53:04.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 74324 25.21 MB/sec warmup 83 sec latency 15.687 ms 2024-02-19T21:53:04.573 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:03.513375+0000 mgr.smithi131.cjubsp (mgr.14164) 353 : cluster [DBG] pgmap v292: 97 pgs: 97 active+clean; 315 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.9 MiB/s wr, 969 op/s 2024-02-19T21:53:04.573 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:03.530061+0000 mgr.smithi131.cjubsp (mgr.14164) 354 : cephadm [INF] Upgrade: Updating mgr.smithi196.voprex 2024-02-19T21:53:04.574 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:03.535222+0000 mon.smithi131 (mon.0) 722 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:04.574 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:03.536069+0000 mon.smithi131 (mon.0) 723 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi196.voprex", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:53:04.574 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:03.536974+0000 mon.smithi131 (mon.0) 724 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:53:04.574 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:03.538128+0000 mon.smithi131 (mon.0) 725 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:04.574 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:03.539599+0000 mgr.smithi131.cjubsp (mgr.14164) 355 : cephadm [INF] Deploying daemon mgr.smithi196.voprex on smithi196 2024-02-19T21:53:05.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 75101 25.16 MB/sec warmup 84 sec latency 14.430 ms 2024-02-19T21:53:05.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:03.513375+0000 mgr.smithi131.cjubsp (mgr.14164) 353 : cluster [DBG] pgmap v292: 97 pgs: 97 active+clean; 315 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.9 MiB/s wr, 969 op/s 2024-02-19T21:53:05.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:03.530061+0000 mgr.smithi131.cjubsp (mgr.14164) 354 : cephadm [INF] Upgrade: Updating mgr.smithi196.voprex 2024-02-19T21:53:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:03.535222+0000 mon.smithi131 (mon.0) 722 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:03.536069+0000 mon.smithi131 (mon.0) 723 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi196.voprex", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:53:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:03.536974+0000 mon.smithi131 (mon.0) 724 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:53:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:03.538128+0000 mon.smithi131 (mon.0) 725 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:03.539599+0000 mgr.smithi131.cjubsp (mgr.14164) 355 : cephadm [INF] Deploying daemon mgr.smithi196.voprex on smithi196 2024-02-19T21:53:05.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:04.878909+0000 mon.smithi131 (mon.0) 726 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:53:05.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:04.886510+0000 mon.smithi131 (mon.0) 727 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:53:06.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 75823 25.02 MB/sec warmup 85 sec latency 26.497 ms 2024-02-19T21:53:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:04.878909+0000 mon.smithi131 (mon.0) 726 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:53:06.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:04.886510+0000 mon.smithi131 (mon.0) 727 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:53:06.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:05.513847+0000 mgr.smithi131.cjubsp (mgr.14164) 356 : cluster [DBG] pgmap v293: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 859 op/s 2024-02-19T21:53:07.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 76410 24.85 MB/sec warmup 86 sec latency 29.776 ms 2024-02-19T21:53:07.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:05.513847+0000 mgr.smithi131.cjubsp (mgr.14164) 356 : cluster [DBG] pgmap v293: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 859 op/s 2024-02-19T21:53:08.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 76962 24.77 MB/sec warmup 87 sec latency 37.790 ms 2024-02-19T21:53:08.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:07.512434+0000 mon.smithi131 (mon.0) 728 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:08.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:07.513198+0000 mon.smithi131 (mon.0) 729 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:08.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:07.514161+0000 mon.smithi131 (mon.0) 730 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:08.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:07.514332+0000 mgr.smithi131.cjubsp (mgr.14164) 357 : cluster [DBG] pgmap v294: 97 pgs: 97 active+clean; 315 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-02-19T21:53:08.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:07.514920+0000 mon.smithi131 (mon.0) 731 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:53:09.001 INFO:teuthology.orchestra.run.smithi131.stdout: 5 77465 24.67 MB/sec warmup 88 sec latency 38.346 ms 2024-02-19T21:53:09.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:07.512434+0000 mon.smithi131 (mon.0) 728 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:09.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:07.513198+0000 mon.smithi131 (mon.0) 729 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:07.514161+0000 mon.smithi131 (mon.0) 730 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:07.514332+0000 mgr.smithi131.cjubsp (mgr.14164) 357 : cluster [DBG] pgmap v294: 97 pgs: 97 active+clean; 315 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-02-19T21:53:09.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:07.514920+0000 mon.smithi131 (mon.0) 731 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:53:09.248 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (6m) 79s ago 7m 21.3M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (7m) 79s ago 7m 6387k - 16.2.5 6933c2a0b7dd 7e4833221591 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (6m) 88s ago 6m 6395k - 16.2.5 6933c2a0b7dd d566245c5426 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (5m) 79s ago 7m 34.1M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (112s) 79s ago 111s 13.6M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (109s) 88s ago 108s 11.8M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:9283 running (9m) 79s ago 9m 422M - 16.2.5 6933c2a0b7dd 785c7b05f4ea 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 starting - - - - 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (9m) 79s ago 9m 44.8M 2048M 16.2.5 6933c2a0b7dd d880a31c6170 2024-02-19T21:53:09.881 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (6m) 88s ago 6m 38.1M 2048M 16.2.5 6933c2a0b7dd 6a272417d66d 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.0.smithi131.nauhyt smithi131 *:2049 running (97s) 79s ago 96s 53.6M - 3.5 6933c2a0b7dd fc874552aefb 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.0.smithi196.hmjmsf smithi196 *:2049 running (91s) 88s ago 91s 53.5M - 3.5 6933c2a0b7dd af920ee7c215 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (7m) 79s ago 7m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (6m) 88s ago 6m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (5m) 79s ago 5m 36.5M 3224M 16.2.5 6933c2a0b7dd f42dfded3a0d 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (5m) 79s ago 5m 38.0M 3224M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (5m) 79s ago 5m 35.1M 3224M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (4m) 79s ago 4m 39.1M 3224M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (4m) 88s ago 4m 35.0M 3992M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (4m) 88s ago 4m 36.8M 3992M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (3m) 88s ago 3m 37.2M 3992M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (3m) 88s ago 3m 39.5M 3992M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:53:09.882 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (5m) 79s ago 6m 40.6M - 2.18.1 de242295e225 22b1466d4ddf 2024-02-19T21:53:10.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 77892 24.59 MB/sec warmup 89 sec latency 35.047 ms 2024-02-19T21:53:10.253 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:53:10.253 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:53:10.253 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:53:10.253 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:53:10.253 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 14 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:53:10.254 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:53:10.568 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:53:10.568 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:53:10.568 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:53:10.568 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [], 2024-02-19T21:53:10.568 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "", 2024-02-19T21:53:10.569 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading mgr daemons" 2024-02-19T21:53:10.569 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:53:11.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 78259 24.43 MB/sec warmup 90 sec latency 42.645 ms 2024-02-19T21:53:11.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:09.235111+0000 mgr.smithi131.cjubsp (mgr.14164) 358 : audit [DBG] from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:11.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:09.514676+0000 mgr.smithi131.cjubsp (mgr.14164) 359 : cluster [DBG] pgmap v295: 97 pgs: 97 active+clean; 315 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 8.3 MiB/s wr, 766 op/s 2024-02-19T21:53:11.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:09.548133+0000 mgr.smithi131.cjubsp (mgr.14164) 360 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:11.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:09.872579+0000 mgr.smithi131.cjubsp (mgr.14164) 361 : audit [DBG] from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:11.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:10.254623+0000 mon.smithi131 (mon.0) 732 : audit [DBG] from='client.? 172.21.15.131:0/477910790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:53:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:09.235111+0000 mgr.smithi131.cjubsp (mgr.14164) 358 : audit [DBG] from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:09.514676+0000 mgr.smithi131.cjubsp (mgr.14164) 359 : cluster [DBG] pgmap v295: 97 pgs: 97 active+clean; 315 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 8.3 MiB/s wr, 766 op/s 2024-02-19T21:53:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:09.548133+0000 mgr.smithi131.cjubsp (mgr.14164) 360 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:09.872579+0000 mgr.smithi131.cjubsp (mgr.14164) 361 : audit [DBG] from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:10.254623+0000 mon.smithi131 (mon.0) 732 : audit [DBG] from='client.? 172.21.15.131:0/477910790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:53:11.916 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:10.569475+0000 mgr.smithi131.cjubsp (mgr.14164) 362 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:12.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 79008 24.36 MB/sec warmup 91 sec latency 34.172 ms 2024-02-19T21:53:12.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:10.569475+0000 mgr.smithi131.cjubsp (mgr.14164) 362 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:13.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 79653 24.22 MB/sec warmup 92 sec latency 54.684 ms 2024-02-19T21:53:13.133 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:11.515202+0000 mgr.smithi131.cjubsp (mgr.14164) 363 : cluster [DBG] pgmap v296: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 928 op/s 2024-02-19T21:53:13.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:11.515202+0000 mgr.smithi131.cjubsp (mgr.14164) 363 : cluster [DBG] pgmap v296: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 928 op/s 2024-02-19T21:53:14.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 80240 24.13 MB/sec warmup 93 sec latency 28.535 ms 2024-02-19T21:53:14.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:13.515618+0000 mgr.smithi131.cjubsp (mgr.14164) 364 : cluster [DBG] pgmap v297: 97 pgs: 97 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 857 op/s 2024-02-19T21:53:14.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:14.271284+0000 mon.smithi131 (mon.0) 733 : cluster [DBG] Standby manager daemon smithi196.voprex restarted 2024-02-19T21:53:14.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:14.271392+0000 mon.smithi131 (mon.0) 734 : cluster [DBG] Standby manager daemon smithi196.voprex started 2024-02-19T21:53:14.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:14.274975+0000 mon.smithi131 (mon.0) 735 : audit [DBG] from='mgr.? 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/crt"}]: dispatch 2024-02-19T21:53:14.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:14.275843+0000 mon.smithi131 (mon.0) 736 : audit [DBG] from='mgr.? 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:53:14.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:14.278384+0000 mon.smithi131 (mon.0) 737 : audit [DBG] from='mgr.? 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/key"}]: dispatch 2024-02-19T21:53:14.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:14.278682+0000 mon.smithi131 (mon.0) 738 : audit [DBG] from='mgr.? 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:53:15.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 81019 24.11 MB/sec warmup 94 sec latency 23.314 ms 2024-02-19T21:53:15.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:13.515618+0000 mgr.smithi131.cjubsp (mgr.14164) 364 : cluster [DBG] pgmap v297: 97 pgs: 97 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 857 op/s 2024-02-19T21:53:15.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:14.271284+0000 mon.smithi131 (mon.0) 733 : cluster [DBG] Standby manager daemon smithi196.voprex restarted 2024-02-19T21:53:15.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:14.271392+0000 mon.smithi131 (mon.0) 734 : cluster [DBG] Standby manager daemon smithi196.voprex started 2024-02-19T21:53:15.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:14.274975+0000 mon.smithi131 (mon.0) 735 : audit [DBG] from='mgr.? 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/crt"}]: dispatch 2024-02-19T21:53:15.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:14.275843+0000 mon.smithi131 (mon.0) 736 : audit [DBG] from='mgr.? 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:53:15.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:14.278384+0000 mon.smithi131 (mon.0) 737 : audit [DBG] from='mgr.? 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/key"}]: dispatch 2024-02-19T21:53:15.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:14.278682+0000 mon.smithi131 (mon.0) 738 : audit [DBG] from='mgr.? 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:53:16.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 81916 24.22 MB/sec warmup 95 sec latency 20.878 ms 2024-02-19T21:53:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:14.848174+0000 mon.smithi131 (mon.0) 739 : cluster [DBG] mgrmap e21: smithi131.cjubsp(active, since 8m), standbys: smithi196.voprex 2024-02-19T21:53:16.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:14.848174+0000 mon.smithi131 (mon.0) 739 : cluster [DBG] mgrmap e21: smithi131.cjubsp(active, since 8m), standbys: smithi196.voprex 2024-02-19T21:53:17.002 INFO:teuthology.orchestra.run.smithi131.stdout: 5 82777 24.13 MB/sec warmup 96 sec latency 44.106 ms 2024-02-19T21:53:17.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:15.516012+0000 mgr.smithi131.cjubsp (mgr.14164) 365 : cluster [DBG] pgmap v298: 97 pgs: 97 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.7 MiB/s wr, 850 op/s 2024-02-19T21:53:17.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:15.516012+0000 mgr.smithi131.cjubsp (mgr.14164) 365 : cluster [DBG] pgmap v298: 97 pgs: 97 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.7 MiB/s wr, 850 op/s 2024-02-19T21:53:18.003 INFO:teuthology.orchestra.run.smithi131.stdout: 5 83585 24.04 MB/sec warmup 97 sec latency 34.592 ms 2024-02-19T21:53:19.003 INFO:teuthology.orchestra.run.smithi131.stdout: 5 84298 23.99 MB/sec warmup 98 sec latency 63.339 ms 2024-02-19T21:53:19.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:17.516719+0000 mgr.smithi131.cjubsp (mgr.14164) 366 : cluster [DBG] pgmap v299: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-02-19T21:53:19.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:17.516719+0000 mgr.smithi131.cjubsp (mgr.14164) 366 : cluster [DBG] pgmap v299: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-02-19T21:53:20.003 INFO:teuthology.orchestra.run.smithi131.stdout: 5 84815 23.95 MB/sec warmup 99 sec latency 31.707 ms 2024-02-19T21:53:21.003 INFO:teuthology.orchestra.run.smithi131.stdout: 5 85276 23.89 MB/sec warmup 100 sec latency 30.668 ms 2024-02-19T21:53:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:19.517079+0000 mgr.smithi131.cjubsp (mgr.14164) 367 : cluster [DBG] pgmap v300: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 839 op/s 2024-02-19T21:53:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:19.921088+0000 mon.smithi131 (mon.0) 740 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:20.264535+0000 mon.smithi131 (mon.0) 741 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:20.271417+0000 mon.smithi131 (mon.0) 742 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:20.273683+0000 mon.smithi131 (mon.0) 743 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:20.276899+0000 mon.smithi131 (mon.0) 744 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr fail", "who": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:53:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:20.281358+0000 mon.smithi131 (mon.0) 745 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-02-19T21:53:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:20.368519+0000 mon.smithi131 (mon.0) 746 : cluster [DBG] Standby manager daemon smithi131.cjubsp started 2024-02-19T21:53:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:19.517079+0000 mgr.smithi131.cjubsp (mgr.14164) 367 : cluster [DBG] pgmap v300: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 839 op/s 2024-02-19T21:53:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:19.921088+0000 mon.smithi131 (mon.0) 740 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:20.264535+0000 mon.smithi131 (mon.0) 741 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:20.271417+0000 mon.smithi131 (mon.0) 742 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' 2024-02-19T21:53:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:20.273683+0000 mon.smithi131 (mon.0) 743 : audit [DBG] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:20.276899+0000 mon.smithi131 (mon.0) 744 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr fail", "who": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:53:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:20.281358+0000 mon.smithi131 (mon.0) 745 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-02-19T21:53:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:20.368519+0000 mon.smithi131 (mon.0) 746 : cluster [DBG] Standby manager daemon smithi131.cjubsp started 2024-02-19T21:53:22.004 INFO:teuthology.orchestra.run.smithi131.stdout: 5 85635 23.75 MB/sec warmup 101 sec latency 43.762 ms 2024-02-19T21:53:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:20.274505+0000 mgr.smithi131.cjubsp (mgr.14164) 368 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi131.cjubsp) 2024-02-19T21:53:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:20.276548+0000 mgr.smithi131.cjubsp (mgr.14164) 369 : cephadm [INF] Failing over to other MGR 2024-02-19T21:53:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.277771+0000 mon.smithi131 (mon.0) 747 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "mgr fail", "who": "smithi131.cjubsp"}]': finished 2024-02-19T21:53:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:21.277844+0000 mon.smithi131 (mon.0) 748 : cluster [DBG] mgrmap e22: smithi196.voprex(active, starting, since 1.00038s), standbys: smithi131.cjubsp 2024-02-19T21:53:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.280478+0000 mon.smithi131 (mon.0) 749 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:53:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.280607+0000 mon.smithi131 (mon.0) 750 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:53:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.280834+0000 mon.smithi131 (mon.0) 751 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.280994+0000 mon.smithi131 (mon.0) 752 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.282301+0000 mon.smithi131 (mon.0) 753 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mgr metadata", "who": "smithi196.voprex", "id": "smithi196.voprex"}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.282442+0000 mon.smithi131 (mon.0) 754 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mgr metadata", "who": "smithi131.cjubsp", "id": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.282572+0000 mon.smithi131 (mon.0) 755 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.282750+0000 mon.smithi131 (mon.0) 756 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.282915+0000 mon.smithi131 (mon.0) 757 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.283074+0000 mon.smithi131 (mon.0) 758 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.283228+0000 mon.smithi131 (mon.0) 759 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.283384+0000 mon.smithi131 (mon.0) 760 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.283539+0000 mon.smithi131 (mon.0) 761 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.283693+0000 mon.smithi131 (mon.0) 762 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.284924+0000 mon.smithi131 (mon.0) 763 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.285098+0000 mon.smithi131 (mon.0) 764 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-19T21:53:22.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.285476+0000 mon.smithi131 (mon.0) 765 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-19T21:53:22.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:21.315456+0000 mon.smithi131 (mon.0) 766 : cluster [INF] Manager daemon smithi196.voprex is now available 2024-02-19T21:53:22.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.335537+0000 mon.smithi131 (mon.0) 767 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:22.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:21.335793+0000 mgr.smithi196.voprex (mgr.14614) 1 : cephadm [INF] Queued nfs.foo for migration 2024-02-19T21:53:22.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.452112+0000 mon.smithi131 (mon.0) 768 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:22.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.457252+0000 mon.smithi131 (mon.0) 769 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:22.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.519908+0000 mon.smithi131 (mon.0) 770 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:22.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.538014+0000 mon.smithi131 (mon.0) 771 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi196.voprex/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:53:22.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:21.545442+0000 mon.smithi131 (mon.0) 772 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi196.voprex/trash_purge_schedule"}]: dispatch 2024-02-19T21:53:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:20.274505+0000 mgr.smithi131.cjubsp (mgr.14164) 368 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi131.cjubsp) 2024-02-19T21:53:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:20.276548+0000 mgr.smithi131.cjubsp (mgr.14164) 369 : cephadm [INF] Failing over to other MGR 2024-02-19T21:53:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.277771+0000 mon.smithi131 (mon.0) 747 : audit [INF] from='mgr.14164 172.21.15.131:0/61131060' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "mgr fail", "who": "smithi131.cjubsp"}]': finished 2024-02-19T21:53:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:21.277844+0000 mon.smithi131 (mon.0) 748 : cluster [DBG] mgrmap e22: smithi196.voprex(active, starting, since 1.00038s), standbys: smithi131.cjubsp 2024-02-19T21:53:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.280478+0000 mon.smithi131 (mon.0) 749 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:53:22.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.280607+0000 mon.smithi131 (mon.0) 750 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.280834+0000 mon.smithi131 (mon.0) 751 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.280994+0000 mon.smithi131 (mon.0) 752 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.282301+0000 mon.smithi131 (mon.0) 753 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mgr metadata", "who": "smithi196.voprex", "id": "smithi196.voprex"}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.282442+0000 mon.smithi131 (mon.0) 754 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mgr metadata", "who": "smithi131.cjubsp", "id": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.282572+0000 mon.smithi131 (mon.0) 755 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.282750+0000 mon.smithi131 (mon.0) 756 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.282915+0000 mon.smithi131 (mon.0) 757 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.283074+0000 mon.smithi131 (mon.0) 758 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.283228+0000 mon.smithi131 (mon.0) 759 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.283384+0000 mon.smithi131 (mon.0) 760 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.283539+0000 mon.smithi131 (mon.0) 761 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.283693+0000 mon.smithi131 (mon.0) 762 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.284924+0000 mon.smithi131 (mon.0) 763 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.285098+0000 mon.smithi131 (mon.0) 764 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-19T21:53:22.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.285476+0000 mon.smithi131 (mon.0) 765 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-19T21:53:22.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:21.315456+0000 mon.smithi131 (mon.0) 766 : cluster [INF] Manager daemon smithi196.voprex is now available 2024-02-19T21:53:22.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.335537+0000 mon.smithi131 (mon.0) 767 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:22.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:21.335793+0000 mgr.smithi196.voprex (mgr.14614) 1 : cephadm [INF] Queued nfs.foo for migration 2024-02-19T21:53:22.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.452112+0000 mon.smithi131 (mon.0) 768 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:22.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.457252+0000 mon.smithi131 (mon.0) 769 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:22.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.519908+0000 mon.smithi131 (mon.0) 770 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:22.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.538014+0000 mon.smithi131 (mon.0) 771 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi196.voprex/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:53:22.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:21.545442+0000 mon.smithi131 (mon.0) 772 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi196.voprex/trash_purge_schedule"}]: dispatch 2024-02-19T21:53:23.004 INFO:teuthology.orchestra.run.smithi131.stdout: 5 86073 23.60 MB/sec warmup 102 sec latency 37.611 ms 2024-02-19T21:53:23.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:22.280713+0000 mon.smithi131 (mon.0) 773 : cluster [DBG] mgrmap e23: smithi196.voprex(active, since 2s), standbys: smithi131.cjubsp 2024-02-19T21:53:23.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:22.297711+0000 mgr.smithi196.voprex (mgr.14614) 2 : cluster [DBG] pgmap v3: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:53:23.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:22.280713+0000 mon.smithi131 (mon.0) 773 : cluster [DBG] mgrmap e23: smithi196.voprex(active, since 2s), standbys: smithi131.cjubsp 2024-02-19T21:53:23.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:22.297711+0000 mgr.smithi196.voprex (mgr.14614) 2 : cluster [DBG] pgmap v3: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:53:24.004 INFO:teuthology.orchestra.run.smithi131.stdout: 5 86642 23.45 MB/sec warmup 103 sec latency 53.390 ms 2024-02-19T21:53:24.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:23.285091+0000 mgr.smithi196.voprex (mgr.14614) 3 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:53:24.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:23.296167+0000 mon.smithi131 (mon.0) 774 : cluster [DBG] mgrmap e24: smithi196.voprex(active, since 3s), standbys: smithi131.cjubsp 2024-02-19T21:53:24.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:23.285091+0000 mgr.smithi196.voprex (mgr.14614) 3 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:53:24.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:23.296167+0000 mon.smithi131 (mon.0) 774 : cluster [DBG] mgrmap e24: smithi196.voprex(active, since 3s), standbys: smithi131.cjubsp 2024-02-19T21:53:25.004 INFO:teuthology.orchestra.run.smithi131.stdout: 5 87170 23.37 MB/sec warmup 104 sec latency 32.765 ms 2024-02-19T21:53:26.004 INFO:teuthology.orchestra.run.smithi131.stdout: 5 87691 23.26 MB/sec warmup 105 sec latency 32.939 ms 2024-02-19T21:53:26.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:25.285444+0000 mgr.smithi196.voprex (mgr.14614) 4 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:53:26.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:25.285444+0000 mgr.smithi196.voprex (mgr.14614) 4 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 318 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:53:27.004 INFO:teuthology.orchestra.run.smithi131.stdout: 5 88298 23.25 MB/sec warmup 106 sec latency 27.044 ms 2024-02-19T21:53:27.970 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:26.968004+0000 mon.smithi131 (mon.0) 775 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:27.970 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:27.285877+0000 mgr.smithi196.voprex (mgr.14614) 5 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 320 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-02-19T21:53:27.970 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:27.289186+0000 mon.smithi131 (mon.0) 776 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:27.970 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:27.290688+0000 mon.smithi131 (mon.0) 777 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:27.970 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:27.292694+0000 mon.smithi131 (mon.0) 778 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:28.004 INFO:teuthology.orchestra.run.smithi131.stdout: 5 88999 23.29 MB/sec warmup 107 sec latency 28.461 ms 2024-02-19T21:53:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:27.293620+0000 mon.smithi131 (mon.0) 779 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:27.294207+0000 mgr.smithi196.voprex (mgr.14614) 6 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 2712M 2024-02-19T21:53:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:27.302105+0000 mon.smithi131 (mon.0) 780 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:26.968004+0000 mon.smithi131 (mon.0) 775 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:27.285877+0000 mgr.smithi196.voprex (mgr.14614) 5 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 320 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-02-19T21:53:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:27.289186+0000 mon.smithi131 (mon.0) 776 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:27.290688+0000 mon.smithi131 (mon.0) 777 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:27.292694+0000 mon.smithi131 (mon.0) 778 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:27.293620+0000 mon.smithi131 (mon.0) 779 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:27.294207+0000 mgr.smithi196.voprex (mgr.14614) 6 : cephadm [INF] Adjusting osd_memory_target on smithi196 to 2712M 2024-02-19T21:53:28.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:27.302105+0000 mon.smithi131 (mon.0) 780 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:28.973 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:28.640209+0000 mon.smithi131 (mon.0) 781 : cluster [DBG] Standby manager daemon smithi131.cjubsp restarted 2024-02-19T21:53:28.973 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:28.640312+0000 mon.smithi131 (mon.0) 782 : cluster [DBG] Standby manager daemon smithi131.cjubsp started 2024-02-19T21:53:28.973 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:28.641597+0000 mon.smithi131 (mon.0) 783 : audit [DBG] from='mgr.? 172.21.15.131:0/285932916' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi131.cjubsp/crt"}]: dispatch 2024-02-19T21:53:28.973 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:28.646062+0000 mon.smithi131 (mon.0) 784 : audit [DBG] from='mgr.? 172.21.15.131:0/285932916' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:53:28.973 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:28.647683+0000 mon.smithi131 (mon.0) 785 : audit [DBG] from='mgr.? 172.21.15.131:0/285932916' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi131.cjubsp/key"}]: dispatch 2024-02-19T21:53:28.974 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:28.648151+0000 mon.smithi131 (mon.0) 786 : audit [DBG] from='mgr.? 172.21.15.131:0/285932916' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:53:29.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 89462 23.13 MB/sec warmup 108 sec latency 43.527 ms 2024-02-19T21:53:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:28.640209+0000 mon.smithi131 (mon.0) 781 : cluster [DBG] Standby manager daemon smithi131.cjubsp restarted 2024-02-19T21:53:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:28.640312+0000 mon.smithi131 (mon.0) 782 : cluster [DBG] Standby manager daemon smithi131.cjubsp started 2024-02-19T21:53:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:28.641597+0000 mon.smithi131 (mon.0) 783 : audit [DBG] from='mgr.? 172.21.15.131:0/285932916' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi131.cjubsp/crt"}]: dispatch 2024-02-19T21:53:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:28.646062+0000 mon.smithi131 (mon.0) 784 : audit [DBG] from='mgr.? 172.21.15.131:0/285932916' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:53:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:28.647683+0000 mon.smithi131 (mon.0) 785 : audit [DBG] from='mgr.? 172.21.15.131:0/285932916' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi131.cjubsp/key"}]: dispatch 2024-02-19T21:53:29.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:28.648151+0000 mon.smithi131 (mon.0) 786 : audit [DBG] from='mgr.? 172.21.15.131:0/285932916' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:53:30.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 90173 23.06 MB/sec warmup 109 sec latency 41.151 ms 2024-02-19T21:53:30.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:28.986512+0000 mon.smithi131 (mon.0) 787 : cluster [DBG] mgrmap e25: smithi196.voprex(active, since 8s), standbys: smithi131.cjubsp 2024-02-19T21:53:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:29.286251+0000 mgr.smithi196.voprex (mgr.14614) 7 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 557 op/s 2024-02-19T21:53:30.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:28.986512+0000 mon.smithi131 (mon.0) 787 : cluster [DBG] mgrmap e25: smithi196.voprex(active, since 8s), standbys: smithi131.cjubsp 2024-02-19T21:53:30.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:29.286251+0000 mgr.smithi196.voprex (mgr.14614) 7 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 557 op/s 2024-02-19T21:53:31.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 90962 23.02 MB/sec warmup 110 sec latency 36.929 ms 2024-02-19T21:53:32.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 91911 23.08 MB/sec warmup 111 sec latency 14.851 ms 2024-02-19T21:53:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:31.286543+0000 mgr.smithi196.voprex (mgr.14614) 8 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.3 MiB/s wr, 433 op/s 2024-02-19T21:53:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:31.858315+0000 mon.smithi131 (mon.0) 788 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.274252+0000 mon.smithi131 (mon.0) 789 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.275077+0000 mon.smithi131 (mon.0) 790 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.275831+0000 mon.smithi131 (mon.0) 791 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.276503+0000 mon.smithi131 (mon.0) 792 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.281094+0000 mon.smithi131 (mon.0) 793 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.281933+0000 mon.smithi131 (mon.0) 794 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:32.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.282590+0000 mon.smithi131 (mon.0) 795 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:53:32.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.333676+0000 mon.smithi131 (mon.0) 796 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:32.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.337413+0000 mon.smithi131 (mon.0) 797 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:31.286543+0000 mgr.smithi196.voprex (mgr.14614) 8 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.3 MiB/s wr, 433 op/s 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:31.858315+0000 mon.smithi131 (mon.0) 788 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.274252+0000 mon.smithi131 (mon.0) 789 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.275077+0000 mon.smithi131 (mon.0) 790 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.275831+0000 mon.smithi131 (mon.0) 791 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.276503+0000 mon.smithi131 (mon.0) 792 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.281094+0000 mon.smithi131 (mon.0) 793 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.281933+0000 mon.smithi131 (mon.0) 794 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.282590+0000 mon.smithi131 (mon.0) 795 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:53:32.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.333676+0000 mon.smithi131 (mon.0) 796 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:32.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.337413+0000 mon.smithi131 (mon.0) 797 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:33.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 92663 23.11 MB/sec warmup 112 sec latency 242.164 ms 2024-02-19T21:53:33.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:32.276884+0000 mgr.smithi196.voprex (mgr.14614) 9 : cephadm [INF] Adjusting osd_memory_target on smithi131 to 1944M 2024-02-19T21:53:33.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:32.283723+0000 mgr.smithi196.voprex (mgr.14614) 10 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.conf 2024-02-19T21:53:33.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:32.283986+0000 mgr.smithi196.voprex (mgr.14614) 11 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.conf 2024-02-19T21:53:33.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:32.302341+0000 mgr.smithi196.voprex (mgr.14614) 12 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:53:33.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:32.303540+0000 mgr.smithi196.voprex (mgr.14614) 13 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:53:33.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.345705+0000 mon.smithi131 (mon.0) 798 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:33.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:32.368084+0000 mon.smithi131 (mon.0) 799 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-02-19T21:53:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:32.276884+0000 mgr.smithi196.voprex (mgr.14614) 9 : cephadm [INF] Adjusting osd_memory_target on smithi131 to 1944M 2024-02-19T21:53:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:32.283723+0000 mgr.smithi196.voprex (mgr.14614) 10 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.conf 2024-02-19T21:53:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:32.283986+0000 mgr.smithi196.voprex (mgr.14614) 11 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.conf 2024-02-19T21:53:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:32.302341+0000 mgr.smithi196.voprex (mgr.14614) 12 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:53:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:32.303540+0000 mgr.smithi196.voprex (mgr.14614) 13 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:53:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.345705+0000 mon.smithi131 (mon.0) 798 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:33.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:32.368084+0000 mon.smithi131 (mon.0) 799 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-02-19T21:53:34.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93215 23.00 MB/sec warmup 113 sec latency 28.031 ms 2024-02-19T21:53:34.669 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:33.287408+0000 mgr.smithi196.voprex (mgr.14614) 14 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 8.5 MiB/s wr, 824 op/s 2024-02-19T21:53:34.669 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:33.352863+0000 mon.smithi131 (mon.0) 800 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-02-19T21:53:34.669 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:33.352933+0000 mon.smithi131 (mon.0) 801 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-02-19T21:53:34.669 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:33.353933+0000 mon.smithi131 (mon.0) 802 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-19T21:53:34.669 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:33.356273+0000 mon.smithi131 (mon.0) 803 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-02-19T21:53:34.669 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:33.356986+0000 mon.smithi131 (mon.0) 804 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-02-19T21:53:34.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:33.287408+0000 mgr.smithi196.voprex (mgr.14614) 14 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 8.5 MiB/s wr, 824 op/s 2024-02-19T21:53:34.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:33.352863+0000 mon.smithi131 (mon.0) 800 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-02-19T21:53:34.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:33.352933+0000 mon.smithi131 (mon.0) 801 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-02-19T21:53:34.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:33.353933+0000 mon.smithi131 (mon.0) 802 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-19T21:53:34.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:33.356273+0000 mon.smithi131 (mon.0) 803 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-02-19T21:53:34.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:33.356986+0000 mon.smithi131 (mon.0) 804 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-02-19T21:53:35.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 22.89 MB/sec warmup 114 sec latency 139.821 ms 2024-02-19T21:53:35.671 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:34.356077+0000 mon.smithi131 (mon.0) 805 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-02-19T21:53:35.671 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:34.356133+0000 mon.smithi131 (mon.0) 806 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-02-19T21:53:35.671 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:34.356232+0000 mon.smithi131 (mon.0) 807 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-02-19T21:53:35.671 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:34.360580+0000 mgr.smithi196.voprex (mgr.14614) 15 : cephadm [INF] Migrating nfs.foo from legacy pool nfs-ganesha namespace foo 2024-02-19T21:53:35.671 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:34.394640+0000 mgr.smithi196.voprex (mgr.14614) 16 : cephadm [INF] Found 1 exports for legacy nfs.foo 2024-02-19T21:53:35.671 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:34.394733+0000 mgr.smithi196.voprex (mgr.14614) 17 : cephadm [INF] Removing old nfs.foo daemons ['nfs.foo.0.0.smithi131.nauhyt', 'nfs.foo.1.0.smithi196.hmjmsf'] 2024-02-19T21:53:35.671 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:34.394797+0000 mgr.smithi196.voprex (mgr.14614) 18 : cephadm [INF] Remove daemons nfs.foo.0.0.smithi131.nauhyt nfs.foo.1.0.smithi196.hmjmsf 2024-02-19T21:53:35.672 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:34.395299+0000 mgr.smithi196.voprex (mgr.14614) 19 : cephadm [INF] Removing daemon nfs.foo.0.0.smithi131.nauhyt from smithi131 -- ports [2049] 2024-02-19T21:53:35.672 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:34.395748+0000 mgr.smithi196.voprex (mgr.14614) 20 : cephadm [INF] Removing daemon nfs.foo.1.0.smithi196.hmjmsf from smithi196 -- ports [2049] 2024-02-19T21:53:35.750 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:34.356077+0000 mon.smithi131 (mon.0) 805 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-02-19T21:53:35.750 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:34.356133+0000 mon.smithi131 (mon.0) 806 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-02-19T21:53:35.751 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:34.356232+0000 mon.smithi131 (mon.0) 807 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-02-19T21:53:35.751 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:34.360580+0000 mgr.smithi196.voprex (mgr.14614) 15 : cephadm [INF] Migrating nfs.foo from legacy pool nfs-ganesha namespace foo 2024-02-19T21:53:35.751 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:34.394640+0000 mgr.smithi196.voprex (mgr.14614) 16 : cephadm [INF] Found 1 exports for legacy nfs.foo 2024-02-19T21:53:35.751 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:34.394733+0000 mgr.smithi196.voprex (mgr.14614) 17 : cephadm [INF] Removing old nfs.foo daemons ['nfs.foo.0.0.smithi131.nauhyt', 'nfs.foo.1.0.smithi196.hmjmsf'] 2024-02-19T21:53:35.751 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:34.394797+0000 mgr.smithi196.voprex (mgr.14614) 18 : cephadm [INF] Remove daemons nfs.foo.0.0.smithi131.nauhyt nfs.foo.1.0.smithi196.hmjmsf 2024-02-19T21:53:35.751 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:34.395299+0000 mgr.smithi196.voprex (mgr.14614) 19 : cephadm [INF] Removing daemon nfs.foo.0.0.smithi131.nauhyt from smithi131 -- ports [2049] 2024-02-19T21:53:35.751 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:34.395748+0000 mgr.smithi196.voprex (mgr.14614) 20 : cephadm [INF] Removing daemon nfs.foo.1.0.smithi196.hmjmsf from smithi196 -- ports [2049] 2024-02-19T21:53:36.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 22.69 MB/sec warmup 115 sec latency 1139.892 ms 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:35.287905+0000 mgr.smithi196.voprex (mgr.14614) 21 : cluster [DBG] pgmap v12: 129 pgs: 32 unknown, 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 8.6 MiB/s wr, 812 op/s 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:35.369151+0000 mon.smithi131 (mon.0) 808 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:35.930069+0000 mon.smithi131 (mon.0) 809 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt"}]: dispatch 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:35.933598+0000 mon.smithi131 (mon.0) 810 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt"}]': finished 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:35.934441+0000 mon.smithi131 (mon.0) 811 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt-rgw"}]: dispatch 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:35.937250+0000 mon.smithi131 (mon.0) 812 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt-rgw"}]': finished 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.342860+0000 mon.smithi131 (mon.0) 813 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf"}]: dispatch 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.346367+0000 mon.smithi131 (mon.0) 814 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf"}]': finished 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.347063+0000 mon.smithi131 (mon.0) 815 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf-rgw"}]: dispatch 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.350077+0000 mon.smithi131 (mon.0) 816 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf-rgw"}]': finished 2024-02-19T21:53:36.523 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.353398+0000 mon.smithi131 (mon.0) 817 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:36.524 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.353798+0000 mon.smithi131 (mon.0) 818 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:35.287905+0000 mgr.smithi196.voprex (mgr.14614) 21 : cluster [DBG] pgmap v12: 129 pgs: 32 unknown, 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 8.6 MiB/s wr, 812 op/s 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:35.369151+0000 mon.smithi131 (mon.0) 808 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:35.930069+0000 mon.smithi131 (mon.0) 809 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt"}]: dispatch 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:35.933598+0000 mon.smithi131 (mon.0) 810 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt"}]': finished 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:35.934441+0000 mon.smithi131 (mon.0) 811 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt-rgw"}]: dispatch 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:35.937250+0000 mon.smithi131 (mon.0) 812 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt-rgw"}]': finished 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.342860+0000 mon.smithi131 (mon.0) 813 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf"}]: dispatch 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.346367+0000 mon.smithi131 (mon.0) 814 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf"}]': finished 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.347063+0000 mon.smithi131 (mon.0) 815 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf-rgw"}]: dispatch 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.350077+0000 mon.smithi131 (mon.0) 816 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf-rgw"}]': finished 2024-02-19T21:53:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.353398+0000 mon.smithi131 (mon.0) 817 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:36.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.353798+0000 mon.smithi131 (mon.0) 818 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2024-02-19T21:53:37.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 22.50 MB/sec warmup 116 sec latency 2139.934 ms 2024-02-19T21:53:37.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:35.929643+0000 mgr.smithi196.voprex (mgr.14614) 22 : cephadm [INF] Removing key for client.nfs.foo.0.0.smithi131.nauhyt 2024-02-19T21:53:37.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:35.934161+0000 mgr.smithi196.voprex (mgr.14614) 23 : cephadm [INF] Removing key for client.nfs.foo.0.0.smithi131.nauhyt-rgw 2024-02-19T21:53:37.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.342212+0000 mgr.smithi196.voprex (mgr.14614) 24 : cephadm [INF] Removing key for client.nfs.foo.1.0.smithi196.hmjmsf 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.346787+0000 mgr.smithi196.voprex (mgr.14614) 25 : cephadm [INF] Removing key for client.nfs.foo.1.0.smithi196.hmjmsf-rgw 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.354315+0000 mgr.smithi196.voprex (mgr.14614) 26 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.384279+0000 mon.smithi131 (mon.0) 819 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.387768+0000 mon.smithi131 (mon.0) 820 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.391892+0000 mgr.smithi196.voprex (mgr.14614) 27 : cephadm [INF] Done migrating nfs.foo 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.391962+0000 mgr.smithi196.voprex (mgr.14614) 28 : cephadm [INF] Done migrating all NFS services 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.395725+0000 mon.smithi131 (mon.0) 821 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.401480+0000 mon.smithi131 (mon.0) 822 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.411555+0000 mon.smithi131 (mon.0) 823 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.422031+0000 mon.smithi131 (mon.0) 824 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.422688+0000 mon.smithi131 (mon.0) 825 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt"}]: dispatch 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.428089+0000 mon.smithi131 (mon.0) 826 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.428596+0000 mon.smithi131 (mon.0) 827 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf"}]: dispatch 2024-02-19T21:53:37.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.433612+0000 mon.smithi131 (mon.0) 828 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.434748+0000 mon.smithi131 (mon.0) 829 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.439607+0000 mon.smithi131 (mon.0) 830 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.440309+0000 mon.smithi131 (mon.0) 831 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.445231+0000 mon.smithi131 (mon.0) 832 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.446229+0000 mon.smithi131 (mon.0) 833 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.483508+0000 mon.smithi131 (mon.0) 834 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.486380+0000 mon.smithi131 (mon.0) 835 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.529846+0000 mon.smithi131 (mon.0) 836 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.534561+0000 mon.smithi131 (mon.0) 837 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-19T21:53:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:36.544485+0000 mon.smithi131 (mon.0) 838 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:37.739 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:35.929643+0000 mgr.smithi196.voprex (mgr.14614) 22 : cephadm [INF] Removing key for client.nfs.foo.0.0.smithi131.nauhyt 2024-02-19T21:53:37.739 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:35.934161+0000 mgr.smithi196.voprex (mgr.14614) 23 : cephadm [INF] Removing key for client.nfs.foo.0.0.smithi131.nauhyt-rgw 2024-02-19T21:53:37.739 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.342212+0000 mgr.smithi196.voprex (mgr.14614) 24 : cephadm [INF] Removing key for client.nfs.foo.1.0.smithi196.hmjmsf 2024-02-19T21:53:37.739 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.346787+0000 mgr.smithi196.voprex (mgr.14614) 25 : cephadm [INF] Removing key for client.nfs.foo.1.0.smithi196.hmjmsf-rgw 2024-02-19T21:53:37.739 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.354315+0000 mgr.smithi196.voprex (mgr.14614) 26 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2024-02-19T21:53:37.739 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.384279+0000 mon.smithi131 (mon.0) 819 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-02-19T21:53:37.739 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.387768+0000 mon.smithi131 (mon.0) 820 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-02-19T21:53:37.739 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.391892+0000 mgr.smithi196.voprex (mgr.14614) 27 : cephadm [INF] Done migrating nfs.foo 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.391962+0000 mgr.smithi196.voprex (mgr.14614) 28 : cephadm [INF] Done migrating all NFS services 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.395725+0000 mon.smithi131 (mon.0) 821 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.401480+0000 mon.smithi131 (mon.0) 822 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.411555+0000 mon.smithi131 (mon.0) 823 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.422031+0000 mon.smithi131 (mon.0) 824 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.422688+0000 mon.smithi131 (mon.0) 825 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.0.0.smithi131.nauhyt"}]: dispatch 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.428089+0000 mon.smithi131 (mon.0) 826 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.428596+0000 mon.smithi131 (mon.0) 827 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.1.0.smithi196.hmjmsf"}]: dispatch 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.433612+0000 mon.smithi131 (mon.0) 828 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.434748+0000 mon.smithi131 (mon.0) 829 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.439607+0000 mon.smithi131 (mon.0) 830 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.440309+0000 mon.smithi131 (mon.0) 831 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-19T21:53:37.740 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.445231+0000 mon.smithi131 (mon.0) 832 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-19T21:53:37.741 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.446229+0000 mon.smithi131 (mon.0) 833 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:37.741 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.483508+0000 mon.smithi131 (mon.0) 834 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:53:37.741 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.486380+0000 mon.smithi131 (mon.0) 835 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:53:37.741 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.529846+0000 mon.smithi131 (mon.0) 836 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:53:37.741 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.534561+0000 mon.smithi131 (mon.0) 837 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-19T21:53:37.741 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:36.544485+0000 mon.smithi131 (mon.0) 838 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:38.005 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 22.31 MB/sec warmup 117 sec latency 3140.024 ms 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.422264+0000 mgr.smithi196.voprex (mgr.14614) 29 : cephadm [INF] Fencing old nfs.foo.0.0.smithi131.nauhyt 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.428267+0000 mgr.smithi196.voprex (mgr.14614) 30 : cephadm [INF] Fencing old nfs.foo.1.0.smithi196.hmjmsf 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.434388+0000 mgr.smithi196.voprex (mgr.14614) 31 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi131.qccypf 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.440018+0000 mgr.smithi196.voprex (mgr.14614) 32 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.529280+0000 mgr.smithi196.voprex (mgr.14614) 33 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.529358+0000 mgr.smithi196.voprex (mgr.14614) 34 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi131.qccypf-rgw 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:36.546252+0000 mgr.smithi196.voprex (mgr.14614) 35 : cephadm [INF] Deploying daemon nfs.foo.0.1.smithi131.qccypf on smithi131 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:37.288620+0000 mgr.smithi196.voprex (mgr.14614) 36 : cluster [DBG] pgmap v14: 129 pgs: 9 unknown, 120 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-02-19T21:53:38.671 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:37.378653+0000 mon.smithi131 (mon.0) 839 : cluster [DBG] mgrmap e26: smithi196.voprex(active, since 17s), standbys: smithi131.cjubsp 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.422264+0000 mgr.smithi196.voprex (mgr.14614) 29 : cephadm [INF] Fencing old nfs.foo.0.0.smithi131.nauhyt 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.428267+0000 mgr.smithi196.voprex (mgr.14614) 30 : cephadm [INF] Fencing old nfs.foo.1.0.smithi196.hmjmsf 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.434388+0000 mgr.smithi196.voprex (mgr.14614) 31 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi131.qccypf 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.440018+0000 mgr.smithi196.voprex (mgr.14614) 32 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.529280+0000 mgr.smithi196.voprex (mgr.14614) 33 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.529358+0000 mgr.smithi196.voprex (mgr.14614) 34 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi131.qccypf-rgw 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:36.546252+0000 mgr.smithi196.voprex (mgr.14614) 35 : cephadm [INF] Deploying daemon nfs.foo.0.1.smithi131.qccypf on smithi131 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:37.288620+0000 mgr.smithi196.voprex (mgr.14614) 36 : cluster [DBG] pgmap v14: 129 pgs: 9 unknown, 120 active+clean; 318 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-02-19T21:53:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:37.378653+0000 mon.smithi131 (mon.0) 839 : cluster [DBG] mgrmap e26: smithi196.voprex(active, since 17s), standbys: smithi131.cjubsp 2024-02-19T21:53:39.006 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 22.12 MB/sec warmup 118 sec latency 4140.105 ms 2024-02-19T21:53:40.006 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 21.93 MB/sec warmup 119 sec latency 5140.184 ms 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:39.062135+0000 mon.smithi131 (mon.0) 840 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:39.063308+0000 mgr.smithi196.voprex (mgr.14614) 37 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi196.rqnuju 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:39.063725+0000 mon.smithi131 (mon.0) 841 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:39.066200+0000 mon.smithi131 (mon.0) 842 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:39.066690+0000 mgr.smithi196.voprex (mgr.14614) 38 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:39.067179+0000 mon.smithi131 (mon.0) 843 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:39.069656+0000 mon.smithi131 (mon.0) 844 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:39.070878+0000 mon.smithi131 (mon.0) 845 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:39.289310+0000 mgr.smithi196.voprex (mgr.14614) 39 : cluster [DBG] pgmap v15: 129 pgs: 129 active+clean; 316 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.4 MiB/s wr, 379 op/s 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:39.062135+0000 mon.smithi131 (mon.0) 840 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:39.063308+0000 mgr.smithi196.voprex (mgr.14614) 37 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi196.rqnuju 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:39.063725+0000 mon.smithi131 (mon.0) 841 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:39.066200+0000 mon.smithi131 (mon.0) 842 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:39.066690+0000 mgr.smithi196.voprex (mgr.14614) 38 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:39.067179+0000 mon.smithi131 (mon.0) 843 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:39.069656+0000 mon.smithi131 (mon.0) 844 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:39.070878+0000 mon.smithi131 (mon.0) 845 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:39.289310+0000 mgr.smithi196.voprex (mgr.14614) 39 : cluster [DBG] pgmap v15: 129 pgs: 129 active+clean; 316 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.4 MiB/s wr, 379 op/s 2024-02-19T21:53:40.888 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (6m) 9s ago 8m 21.2M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (8m) 9s ago 8m 6387k - 16.2.5 6933c2a0b7dd 7e4833221591 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (7m) 14s ago 7m 6395k - 16.2.5 6933c2a0b7dd d566245c5426 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (6m) 9s ago 7m 34.2M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (2m) 9s ago 2m 39.9M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (2m) 14s ago 2m 12.4M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:9283 running (9m) 9s ago 9m 64.3M - 16.2.5 6933c2a0b7dd 785c7b05f4ea 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (34s) 14s ago 7m 414M - 16.2.14-548-g81bd20d6 823b921721f5 c819a4544358 2024-02-19T21:53:41.451 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (9m) 9s ago 9m 53.9M 2048M 16.2.5 6933c2a0b7dd d880a31c6170 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (7m) 14s ago 7m 47.7M 2048M 16.2.5 6933c2a0b7dd 6a272417d66d 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 starting - - - - 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (7m) 9s ago 7m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (6m) 14s ago 6m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (6m) 9s ago 6m 342M 1944M 16.2.5 6933c2a0b7dd f42dfded3a0d 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (6m) 9s ago 6m 265M 1944M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (5m) 9s ago 5m 207M 1944M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (5m) 9s ago 5m 298M 1944M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (5m) 14s ago 5m 342M 2712M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (4m) 14s ago 4m 353M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (4m) 14s ago 4m 288M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (4m) 14s ago 4m 440M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:53:41.452 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (6m) 9s ago 7m 53.5M - 2.18.1 de242295e225 22b1466d4ddf 2024-02-19T21:53:41.794 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 1, 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:53:41.795 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:53:41.796 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:53:41.796 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 1, 2024-02-19T21:53:41.796 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 13 2024-02-19T21:53:41.796 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:53:41.796 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:53:42.006 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 0.00 MB/sec execute 1 sec latency 7140.268 ms 2024-02-19T21:53:42.088 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:53:42.089 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:53:42.089 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:53:42.089 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:53:42.089 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [], 2024-02-19T21:53:42.089 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "1/22 daemons upgraded", 2024-02-19T21:53:42.089 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "", 2024-02-19T21:53:42.089 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:53:42.089 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:53:42.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:40.874788+0000 mgr.smithi196.voprex (mgr.14614) 40 : audit [DBG] from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:42.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:41.160221+0000 mgr.smithi196.voprex (mgr.14614) 41 : audit [DBG] from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:42.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:41.289834+0000 mgr.smithi196.voprex (mgr.14614) 42 : cluster [DBG] pgmap v16: 129 pgs: 129 active+clean; 316 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.6 MiB/s wr, 286 op/s 2024-02-19T21:53:42.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:41.795764+0000 mon.smithi131 (mon.0) 846 : audit [DBG] from='client.? 172.21.15.131:0/248594117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:53:42.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:41.956441+0000 mon.smithi131 (mon.0) 847 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:53:42.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:41.959961+0000 mon.smithi131 (mon.0) 848 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:53:42.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:41.968860+0000 mon.smithi131 (mon.0) 849 : audit [INF] from='mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639]' entity='mds.foofs.smithi131.gemwnz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.131:0/1327093367"}]: dispatch 2024-02-19T21:53:42.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:42.011086+0000 mon.smithi131 (mon.0) 850 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:53:42.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:42.014755+0000 mon.smithi131 (mon.0) 851 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-19T21:53:42.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:42.015905+0000 mon.smithi131 (mon.0) 852 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:40.874788+0000 mgr.smithi196.voprex (mgr.14614) 40 : audit [DBG] from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:41.160221+0000 mgr.smithi196.voprex (mgr.14614) 41 : audit [DBG] from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:41.289834+0000 mgr.smithi196.voprex (mgr.14614) 42 : cluster [DBG] pgmap v16: 129 pgs: 129 active+clean; 316 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.6 MiB/s wr, 286 op/s 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:41.795764+0000 mon.smithi131 (mon.0) 846 : audit [DBG] from='client.? 172.21.15.131:0/248594117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:41.956441+0000 mon.smithi131 (mon.0) 847 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:41.959961+0000 mon.smithi131 (mon.0) 848 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:41.968860+0000 mon.smithi131 (mon.0) 849 : audit [INF] from='mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639]' entity='mds.foofs.smithi131.gemwnz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.131:0/1327093367"}]: dispatch 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:42.011086+0000 mon.smithi131 (mon.0) 850 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:42.014755+0000 mon.smithi131 (mon.0) 851 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-19T21:53:42.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:42.015905+0000 mon.smithi131 (mon.0) 852 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:43.006 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 0.00 MB/sec execute 2 sec latency 8140.487 ms 2024-02-19T21:53:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:41.441702+0000 mgr.smithi196.voprex (mgr.14614) 43 : audit [DBG] from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:41.968661+0000 mds.foofs.smithi131.gemwnz (mds.0) 1 : cluster [INF] Evicting (and blocklisting) client session 14528 (172.21.15.131:0/1327093367) 2024-02-19T21:53:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:41.973494+0000 mds.foofs.smithi131.gemwnz (mds.0) 2 : cluster [WRN] client session with duplicated session uuid 'ganesha-nfs.foo.0-0001' denied (client.14696 172.21.15.131:0/2371637368) 2024-02-19T21:53:43.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:42.010539+0000 mgr.smithi196.voprex (mgr.14614) 44 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:53:43.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:42.010662+0000 mgr.smithi196.voprex (mgr.14614) 45 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi196.rqnuju-rgw 2024-02-19T21:53:43.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:42.016593+0000 mgr.smithi196.voprex (mgr.14614) 46 : cephadm [INF] Deploying daemon nfs.foo.1.1.smithi196.rqnuju on smithi196 2024-02-19T21:53:43.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:42.088794+0000 mgr.smithi196.voprex (mgr.14614) 47 : audit [DBG] from='client.14692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:43.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:42.962830+0000 mon.smithi131 (mon.0) 853 : audit [INF] from='mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639]' entity='mds.foofs.smithi131.gemwnz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.131:0/1327093367"}]': finished 2024-02-19T21:53:43.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:42.962901+0000 mon.smithi131 (mon.0) 854 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-02-19T21:53:43.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:41.441702+0000 mgr.smithi196.voprex (mgr.14614) 43 : audit [DBG] from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:43.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:41.968661+0000 mds.foofs.smithi131.gemwnz (mds.0) 1 : cluster [INF] Evicting (and blocklisting) client session 14528 (172.21.15.131:0/1327093367) 2024-02-19T21:53:43.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:41.973494+0000 mds.foofs.smithi131.gemwnz (mds.0) 2 : cluster [WRN] client session with duplicated session uuid 'ganesha-nfs.foo.0-0001' denied (client.14696 172.21.15.131:0/2371637368) 2024-02-19T21:53:43.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:42.010539+0000 mgr.smithi196.voprex (mgr.14614) 44 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:53:43.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:42.010662+0000 mgr.smithi196.voprex (mgr.14614) 45 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi196.rqnuju-rgw 2024-02-19T21:53:43.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:42.016593+0000 mgr.smithi196.voprex (mgr.14614) 46 : cephadm [INF] Deploying daemon nfs.foo.1.1.smithi196.rqnuju on smithi196 2024-02-19T21:53:43.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:42.088794+0000 mgr.smithi196.voprex (mgr.14614) 47 : audit [DBG] from='client.14692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:53:43.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:42.962830+0000 mon.smithi131 (mon.0) 853 : audit [INF] from='mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639]' entity='mds.foofs.smithi131.gemwnz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.131:0/1327093367"}]': finished 2024-02-19T21:53:43.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:42.962901+0000 mon.smithi131 (mon.0) 854 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-02-19T21:53:44.006 INFO:teuthology.orchestra.run.smithi131.stdout: 5 93775 0.00 MB/sec execute 3 sec latency 9140.530 ms 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:[93184] open ./clients/client4/~dmtmp/PWRPNT/NEWTIPS.PPT failed for handle 12844 (Stale file handle) 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:(93185) ERROR: handle 12844 was not found 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:[93773] open ./clients/client1/~dmtmp/PARADOX/CHANGED.DB failed for handle 12954 (Stale file handle) 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:(93774) ERROR: handle 12954 was not found 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:[93190] open ./clients/client0/~dmtmp/PWRPNT/NEWTIPS.PPT failed for handle 12845 (Stale file handle) 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:(93191) ERROR: handle 12845 was not found 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:[94124] open ./clients/client2/~dmtmp/PARADOX/__43212.DB failed for handle 13056 (Stale file handle) 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:(94125) ERROR: handle 13056 was not found 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:Child failed with status 1 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:[94605] unlink ./clients/client3/~dmtmp/EXCEL/RESULTS.XLS failed (Stale file handle) - expected NT_STATUS_OK 2024-02-19T21:53:44.239 INFO:teuthology.orchestra.run.smithi131.stdout:ERROR: child 3 failed at line 94605 2024-02-19T21:53:44.241 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-02-19T21:53:44.300 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-02-19T21:53:44.426 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:53:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:43.290488+0000 mgr.smithi196.voprex (mgr.14614) 48 : cluster [DBG] pgmap v18: 129 pgs: 129 active+clean; 313 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 2.6 MiB/s wr, 290 op/s 2024-02-19T21:53:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:43.290488+0000 mgr.smithi196.voprex (mgr.14614) 48 : cluster [DBG] pgmap v18: 129 pgs: 129 active+clean; 313 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 2.6 MiB/s wr, 290 op/s 2024-02-19T21:53:46.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:45.290849+0000 mgr.smithi196.voprex (mgr.14614) 49 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 313 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.1 MiB/s wr, 234 op/s 2024-02-19T21:53:46.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:46.115451+0000 mon.smithi131 (mon.0) 855 : audit [INF] from='mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639]' entity='mds.foofs.smithi131.gemwnz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.196:0/4236736864"}]: dispatch 2024-02-19T21:53:46.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:46.223619+0000 mon.smithi131 (mon.0) 856 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:46.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:46.225645+0000 mon.smithi131 (mon.0) 857 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:46.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:45.290849+0000 mgr.smithi196.voprex (mgr.14614) 49 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 313 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.1 MiB/s wr, 234 op/s 2024-02-19T21:53:46.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:46.115451+0000 mon.smithi131 (mon.0) 855 : audit [INF] from='mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639]' entity='mds.foofs.smithi131.gemwnz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.196:0/4236736864"}]: dispatch 2024-02-19T21:53:46.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:46.223619+0000 mon.smithi131 (mon.0) 856 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:46.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:46.225645+0000 mon.smithi131 (mon.0) 857 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:53:47.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:46.115220+0000 mds.foofs.smithi131.gemwnz (mds.0) 3 : cluster [INF] Evicting (and blocklisting) client session 24369 (172.21.15.196:0/4236736864) 2024-02-19T21:53:47.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:46.119877+0000 mds.foofs.smithi131.gemwnz (mds.0) 4 : cluster [WRN] client session with duplicated session uuid 'ganesha-nfs.foo.1-0001' denied (client.24523 172.21.15.196:0/1562926069) 2024-02-19T21:53:47.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:46.354138+0000 mon.smithi131 (mon.0) 858 : audit [INF] from='mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639]' entity='mds.foofs.smithi131.gemwnz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.196:0/4236736864"}]': finished 2024-02-19T21:53:47.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:46.354203+0000 mon.smithi131 (mon.0) 859 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-02-19T21:53:47.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:46.521625+0000 mon.smithi131 (mon.0) 860 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:47.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:46.115220+0000 mds.foofs.smithi131.gemwnz (mds.0) 3 : cluster [INF] Evicting (and blocklisting) client session 24369 (172.21.15.196:0/4236736864) 2024-02-19T21:53:47.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:46.119877+0000 mds.foofs.smithi131.gemwnz (mds.0) 4 : cluster [WRN] client session with duplicated session uuid 'ganesha-nfs.foo.1-0001' denied (client.24523 172.21.15.196:0/1562926069) 2024-02-19T21:53:47.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:46.354138+0000 mon.smithi131 (mon.0) 858 : audit [INF] from='mds.? [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639]' entity='mds.foofs.smithi131.gemwnz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.196:0/4236736864"}]': finished 2024-02-19T21:53:47.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:46.354203+0000 mon.smithi131 (mon.0) 859 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-02-19T21:53:47.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:46.521625+0000 mon.smithi131 (mon.0) 860 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:48.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:47.291409+0000 mgr.smithi196.voprex (mgr.14614) 50 : cluster [DBG] pgmap v21: 129 pgs: 129 active+clean; 307 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 118 KiB/s wr, 30 op/s 2024-02-19T21:53:48.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:47.291409+0000 mgr.smithi196.voprex (mgr.14614) 50 : cluster [DBG] pgmap v21: 129 pgs: 129 active+clean; 307 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 118 KiB/s wr, 30 op/s 2024-02-19T21:53:49.464 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:53:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:49.292011+0000 mgr.smithi196.voprex (mgr.14614) 51 : cluster [DBG] pgmap v22: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 118 KiB/s wr, 39 op/s 2024-02-19T21:53:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:49.292011+0000 mgr.smithi196.voprex (mgr.14614) 51 : cluster [DBG] pgmap v22: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 118 KiB/s wr, 39 op/s 2024-02-19T21:53:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:50.681314+0000 mon.smithi131 (mon.0) 861 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:51.292590+0000 mgr.smithi196.voprex (mgr.14614) 52 : cluster [DBG] pgmap v23: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 47 KiB/s wr, 31 op/s 2024-02-19T21:53:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:50.681314+0000 mon.smithi131 (mon.0) 861 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:51.292590+0000 mgr.smithi196.voprex (mgr.14614) 52 : cluster [DBG] pgmap v23: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 47 KiB/s wr, 31 op/s 2024-02-19T21:53:53.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:52.191313+0000 mon.smithi131 (mon.0) 862 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:53.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:52.192381+0000 mon.smithi131 (mon.0) 863 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:53.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:52.193206+0000 mon.smithi131 (mon.0) 864 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:53:53.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:52.197660+0000 mon.smithi131 (mon.0) 865 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:53.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:53:52.204039+0000 mon.smithi131 (mon.0) 866 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:53.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:52.206020+0000 mgr.smithi196.voprex (mgr.14614) 53 : cephadm [INF] Reconfiguring prometheus.smithi131 (dependencies changed)... 2024-02-19T21:53:53.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:53:52.214853+0000 mgr.smithi196.voprex (mgr.14614) 54 : cephadm [INF] Reconfiguring daemon prometheus.smithi131 on smithi131 2024-02-19T21:53:53.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:52.191313+0000 mon.smithi131 (mon.0) 862 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:53.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:52.192381+0000 mon.smithi131 (mon.0) 863 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:53:53.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:52.193206+0000 mon.smithi131 (mon.0) 864 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:53:53.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:52.197660+0000 mon.smithi131 (mon.0) 865 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:53.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:53:52.204039+0000 mon.smithi131 (mon.0) 866 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:53:53.677 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:52.206020+0000 mgr.smithi196.voprex (mgr.14614) 53 : cephadm [INF] Reconfiguring prometheus.smithi131 (dependencies changed)... 2024-02-19T21:53:53.677 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:53:52.214853+0000 mgr.smithi196.voprex (mgr.14614) 54 : cephadm [INF] Reconfiguring daemon prometheus.smithi131 on smithi131 2024-02-19T21:53:54.495 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:53:54.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:53.293426+0000 mgr.smithi196.voprex (mgr.14614) 55 : cluster [DBG] pgmap v24: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 42 KiB/s wr, 27 op/s 2024-02-19T21:53:54.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:53.293426+0000 mgr.smithi196.voprex (mgr.14614) 55 : cluster [DBG] pgmap v24: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 42 KiB/s wr, 27 op/s 2024-02-19T21:53:56.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:55.293849+0000 mgr.smithi196.voprex (mgr.14614) 56 : cluster [DBG] pgmap v25: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 42 KiB/s wr, 27 op/s 2024-02-19T21:53:56.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:55.293849+0000 mgr.smithi196.voprex (mgr.14614) 56 : cluster [DBG] pgmap v25: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 42 KiB/s wr, 27 op/s 2024-02-19T21:53:58.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:57.294527+0000 mgr.smithi196.voprex (mgr.14614) 57 : cluster [DBG] pgmap v26: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-02-19T21:53:58.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:57.294527+0000 mgr.smithi196.voprex (mgr.14614) 57 : cluster [DBG] pgmap v26: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 38 KiB/s wr, 25 op/s 2024-02-19T21:53:59.540 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:00.095 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:53:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:53:59.295133+0000 mgr.smithi196.voprex (mgr.14614) 58 : cluster [DBG] pgmap v27: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 2.3 KiB/s wr, 7 op/s 2024-02-19T21:54:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:53:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:53:59.295133+0000 mgr.smithi196.voprex (mgr.14614) 58 : cluster [DBG] pgmap v27: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 2.3 KiB/s wr, 7 op/s 2024-02-19T21:54:02.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:01.295497+0000 mgr.smithi196.voprex (mgr.14614) 59 : cluster [DBG] pgmap v28: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 2.3 KiB/s wr, 1 op/s 2024-02-19T21:54:02.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:01.295497+0000 mgr.smithi196.voprex (mgr.14614) 59 : cluster [DBG] pgmap v28: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 2.3 KiB/s wr, 1 op/s 2024-02-19T21:54:04.579 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:04.662 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:03.296267+0000 mgr.smithi196.voprex (mgr.14614) 60 : cluster [DBG] pgmap v29: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 2.7 KiB/s wr, 1 op/s 2024-02-19T21:54:04.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:03.296267+0000 mgr.smithi196.voprex (mgr.14614) 60 : cluster [DBG] pgmap v29: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 2.7 KiB/s wr, 1 op/s 2024-02-19T21:54:06.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:05.296814+0000 mgr.smithi196.voprex (mgr.14614) 61 : cluster [DBG] pgmap v30: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T21:54:06.702 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:05.296814+0000 mgr.smithi196.voprex (mgr.14614) 61 : cluster [DBG] pgmap v30: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T21:54:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:07.297653+0000 mgr.smithi196.voprex (mgr.14614) 62 : cluster [DBG] pgmap v31: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 0 op/s 2024-02-19T21:54:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:07.519999+0000 mon.smithi131 (mon.0) 867 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:07.524355+0000 mon.smithi131 (mon.0) 868 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T21:54:08.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:07.526607+0000 mon.smithi131 (mon.0) 869 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi131.front.sepia.ceph.com:9095"}]: dispatch 2024-02-19T21:54:08.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:07.530382+0000 mon.smithi131 (mon.0) 870 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:08.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:07.534204+0000 mon.smithi131 (mon.0) 871 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:08.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:07.539713+0000 mon.smithi131 (mon.0) 872 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:08.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:07.297653+0000 mgr.smithi196.voprex (mgr.14614) 62 : cluster [DBG] pgmap v31: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 0 op/s 2024-02-19T21:54:08.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:07.519999+0000 mon.smithi131 (mon.0) 867 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:08.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:07.524355+0000 mon.smithi131 (mon.0) 868 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T21:54:08.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:07.526607+0000 mon.smithi131 (mon.0) 869 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi131.front.sepia.ceph.com:9095"}]: dispatch 2024-02-19T21:54:08.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:07.530382+0000 mon.smithi131 (mon.0) 870 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:08.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:07.534204+0000 mon.smithi131 (mon.0) 871 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:08.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:07.539713+0000 mon.smithi131 (mon.0) 872 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:09.620 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:09.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:07.524769+0000 mgr.smithi196.voprex (mgr.14614) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T21:54:09.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:07.527012+0000 mgr.smithi196.voprex (mgr.14614) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi131.front.sepia.ceph.com:9095"}]: dispatch 2024-02-19T21:54:09.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:07.540741+0000 mgr.smithi196.voprex (mgr.14614) 65 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi196.voprex) 2024-02-19T21:54:09.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:07.542234+0000 mgr.smithi196.voprex (mgr.14614) 66 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi196.voprex) 2024-02-19T21:54:09.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:08.964829+0000 mon.smithi131 (mon.0) 873 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:09.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:08.965638+0000 mon.smithi131 (mon.0) 874 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi131.cjubsp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:54:09.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:08.966525+0000 mon.smithi131 (mon.0) 875 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:54:09.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:08.967348+0000 mon.smithi131 (mon.0) 876 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:09.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:07.524769+0000 mgr.smithi196.voprex (mgr.14614) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T21:54:09.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:07.527012+0000 mgr.smithi196.voprex (mgr.14614) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi131.front.sepia.ceph.com:9095"}]: dispatch 2024-02-19T21:54:09.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:07.540741+0000 mgr.smithi196.voprex (mgr.14614) 65 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi196.voprex) 2024-02-19T21:54:09.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:07.542234+0000 mgr.smithi196.voprex (mgr.14614) 66 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi196.voprex) 2024-02-19T21:54:09.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:08.964829+0000 mon.smithi131 (mon.0) 873 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:09.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:08.965638+0000 mon.smithi131 (mon.0) 874 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi131.cjubsp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:54:09.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:08.966525+0000 mon.smithi131 (mon.0) 875 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:54:09.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:08.967348+0000 mon.smithi131 (mon.0) 876 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:10.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:08.960137+0000 mgr.smithi196.voprex (mgr.14614) 67 : cephadm [INF] Upgrade: Updating mgr.smithi131.cjubsp 2024-02-19T21:54:10.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:08.967973+0000 mgr.smithi196.voprex (mgr.14614) 68 : cephadm [INF] Deploying daemon mgr.smithi131.cjubsp on smithi131 2024-02-19T21:54:10.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:09.298138+0000 mgr.smithi196.voprex (mgr.14614) 69 : cluster [DBG] pgmap v32: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T21:54:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:08.960137+0000 mgr.smithi196.voprex (mgr.14614) 67 : cephadm [INF] Upgrade: Updating mgr.smithi131.cjubsp 2024-02-19T21:54:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:08.967973+0000 mgr.smithi196.voprex (mgr.14614) 68 : cephadm [INF] Deploying daemon mgr.smithi131.cjubsp on smithi131 2024-02-19T21:54:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:09.298138+0000 mgr.smithi196.voprex (mgr.14614) 69 : cluster [DBG] pgmap v32: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T21:54:12.416 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:54:12.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:11.298600+0000 mgr.smithi196.voprex (mgr.14614) 70 : cluster [DBG] pgmap v33: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T21:54:12.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:11.298600+0000 mgr.smithi196.voprex (mgr.14614) 70 : cluster [DBG] pgmap v33: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T21:54:13.051 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (7m) 20s ago 8m 21.2M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (8m) 20s ago 8m 6387k - 16.2.5 6933c2a0b7dd 7e4833221591 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (7m) 22s ago 7m 6395k - 16.2.5 6933c2a0b7dd d566245c5426 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (6m) 20s ago 8m 34.2M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (2m) 20s ago 2m 38.8M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (2m) 22s ago 2m 12.6M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 starting - - - - 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (66s) 22s ago 7m 431M - 16.2.14-548-g81bd20d6 823b921721f5 c819a4544358 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (10m) 20s ago 10m 51.0M 2048M 16.2.5 6933c2a0b7dd d880a31c6170 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (7m) 22s ago 7m 40.4M 2048M 16.2.5 6933c2a0b7dd 6a272417d66d 2024-02-19T21:54:13.052 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (34s) 20s ago 34s 49.5M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (27s) 22s ago 26s 48.8M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (8m) 20s ago 8m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (7m) 22s ago 7m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (7m) 20s ago 7m 373M 1944M 16.2.5 6933c2a0b7dd f42dfded3a0d 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (6m) 20s ago 6m 285M 1944M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (6m) 20s ago 6m 226M 1944M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (5m) 20s ago 5m 340M 1944M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (5m) 22s ago 5m 356M 2712M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (5m) 22s ago 5m 374M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (4m) 22s ago 4m 311M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (4m) 22s ago 4m 466M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:54:13.053 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 starting - - - - 2024-02-19T21:54:13.416 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:54:13.416 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:54:13.416 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:54:13.416 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:54:13.416 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:54:13.416 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 1, 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 1, 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 13 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:54:13.417 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:54:13.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:12.394030+0000 mon.smithi131 (mon.0) 877 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:13.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:12.395217+0000 mon.smithi131 (mon.0) 878 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:13.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:12.401636+0000 mgr.smithi196.voprex (mgr.14614) 71 : audit [DBG] from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:13.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:12.394030+0000 mon.smithi131 (mon.0) 877 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:13.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:12.395217+0000 mon.smithi131 (mon.0) 878 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:13.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:12.401636+0000 mgr.smithi196.voprex (mgr.14614) 71 : audit [DBG] from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [], 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "", 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading mgr daemons", 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:54:13.773 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:54:14.667 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:14.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:12.736628+0000 mgr.smithi196.voprex (mgr.14614) 72 : audit [DBG] from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:14.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:13.043196+0000 mgr.smithi196.voprex (mgr.14614) 73 : audit [DBG] from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:14.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:13.299304+0000 mgr.smithi196.voprex (mgr.14614) 74 : cluster [DBG] pgmap v34: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-02-19T21:54:14.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:13.416773+0000 mon.smithi196 (mon.1) 18 : audit [DBG] from='client.? 172.21.15.131:0/586171470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:54:14.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:12.736628+0000 mgr.smithi196.voprex (mgr.14614) 72 : audit [DBG] from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:14.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:13.043196+0000 mgr.smithi196.voprex (mgr.14614) 73 : audit [DBG] from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:14.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:13.299304+0000 mgr.smithi196.voprex (mgr.14614) 74 : cluster [DBG] pgmap v34: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-02-19T21:54:14.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:13.416773+0000 mon.smithi196 (mon.1) 18 : audit [DBG] from='client.? 172.21.15.131:0/586171470' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:54:15.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:13.773907+0000 mgr.smithi196.voprex (mgr.14614) 75 : audit [DBG] from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:15.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:13.773907+0000 mgr.smithi196.voprex (mgr.14614) 75 : audit [DBG] from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:16.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:15.299676+0000 mgr.smithi196.voprex (mgr.14614) 76 : cluster [DBG] pgmap v35: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:16.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:15.299676+0000 mgr.smithi196.voprex (mgr.14614) 76 : cluster [DBG] pgmap v35: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:18.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:17.300394+0000 mgr.smithi196.voprex (mgr.14614) 77 : cluster [DBG] pgmap v36: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:18.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:17.961950+0000 mon.smithi131 (mon.0) 879 : cluster [DBG] Standby manager daemon smithi131.cjubsp restarted 2024-02-19T21:54:18.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:17.962046+0000 mon.smithi131 (mon.0) 880 : cluster [DBG] Standby manager daemon smithi131.cjubsp started 2024-02-19T21:54:18.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:17.964119+0000 mon.smithi131 (mon.0) 881 : audit [DBG] from='mgr.? 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi131.cjubsp/crt"}]: dispatch 2024-02-19T21:54:18.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:17.964610+0000 mon.smithi131 (mon.0) 882 : audit [DBG] from='mgr.? 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:54:18.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:17.966528+0000 mon.smithi131 (mon.0) 883 : audit [DBG] from='mgr.? 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi131.cjubsp/key"}]: dispatch 2024-02-19T21:54:18.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:17.967366+0000 mon.smithi131 (mon.0) 884 : audit [DBG] from='mgr.? 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:54:18.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:17.300394+0000 mgr.smithi196.voprex (mgr.14614) 77 : cluster [DBG] pgmap v36: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:18.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:17.961950+0000 mon.smithi131 (mon.0) 879 : cluster [DBG] Standby manager daemon smithi131.cjubsp restarted 2024-02-19T21:54:18.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:17.962046+0000 mon.smithi131 (mon.0) 880 : cluster [DBG] Standby manager daemon smithi131.cjubsp started 2024-02-19T21:54:18.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:17.964119+0000 mon.smithi131 (mon.0) 881 : audit [DBG] from='mgr.? 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi131.cjubsp/crt"}]: dispatch 2024-02-19T21:54:18.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:17.964610+0000 mon.smithi131 (mon.0) 882 : audit [DBG] from='mgr.? 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:54:18.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:17.966528+0000 mon.smithi131 (mon.0) 883 : audit [DBG] from='mgr.? 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi131.cjubsp/key"}]: dispatch 2024-02-19T21:54:18.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:17.967366+0000 mon.smithi131 (mon.0) 884 : audit [DBG] from='mgr.? 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:54:19.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:18.417007+0000 mon.smithi131 (mon.0) 885 : cluster [DBG] mgrmap e27: smithi196.voprex(active, since 58s), standbys: smithi131.cjubsp 2024-02-19T21:54:19.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:19.133910+0000 mon.smithi131 (mon.0) 886 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:19.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:19.300956+0000 mgr.smithi196.voprex (mgr.14614) 78 : cluster [DBG] pgmap v37: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:19.704 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:19.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:18.417007+0000 mon.smithi131 (mon.0) 885 : cluster [DBG] mgrmap e27: smithi196.voprex(active, since 58s), standbys: smithi131.cjubsp 2024-02-19T21:54:19.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:19.133910+0000 mon.smithi131 (mon.0) 886 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:19.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:19.300956+0000 mgr.smithi196.voprex (mgr.14614) 78 : cluster [DBG] pgmap v37: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:20.736 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:19.474521+0000 mon.smithi131 (mon.0) 887 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:20.736 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:19.475592+0000 mon.smithi131 (mon.0) 888 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:20.736 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:19.476431+0000 mon.smithi131 (mon.0) 889 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:54:20.737 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:19.480909+0000 mon.smithi131 (mon.0) 890 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:20.737 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:19.486456+0000 mon.smithi131 (mon.0) 891 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:20.737 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:19.489856+0000 mon.smithi131 (mon.0) 892 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:20.737 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:19.492139+0000 mgr.smithi196.voprex (mgr.14614) 79 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi196.voprex) 2024-02-19T21:54:20.737 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:19.492586+0000 mgr.smithi196.voprex (mgr.14614) 80 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi196.voprex) 2024-02-19T21:54:20.737 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:19.494774+0000 mgr.smithi196.voprex (mgr.14614) 81 : cephadm [INF] Failing over to other MGR 2024-02-19T21:54:20.737 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:19.495372+0000 mon.smithi131 (mon.0) 893 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mgr fail", "who": "smithi196.voprex"}]: dispatch 2024-02-19T21:54:20.737 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:19.498659+0000 mon.smithi131 (mon.0) 894 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-02-19T21:54:20.741 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:19.474521+0000 mon.smithi131 (mon.0) 887 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:20.741 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:19.475592+0000 mon.smithi131 (mon.0) 888 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:20.741 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:19.476431+0000 mon.smithi131 (mon.0) 889 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:54:20.741 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:19.480909+0000 mon.smithi131 (mon.0) 890 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:20.741 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:19.486456+0000 mon.smithi131 (mon.0) 891 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' 2024-02-19T21:54:20.741 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:19.489856+0000 mon.smithi131 (mon.0) 892 : audit [DBG] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:20.741 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:19.492139+0000 mgr.smithi196.voprex (mgr.14614) 79 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi196.voprex) 2024-02-19T21:54:20.741 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:19.492586+0000 mgr.smithi196.voprex (mgr.14614) 80 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi196.voprex) 2024-02-19T21:54:20.742 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:19.494774+0000 mgr.smithi196.voprex (mgr.14614) 81 : cephadm [INF] Failing over to other MGR 2024-02-19T21:54:20.742 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:19.495372+0000 mon.smithi131 (mon.0) 893 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd=[{"prefix": "mgr fail", "who": "smithi196.voprex"}]: dispatch 2024-02-19T21:54:20.742 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:19.498659+0000 mon.smithi131 (mon.0) 894 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-02-19T21:54:21.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.491654+0000 mon.smithi131 (mon.0) 895 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "mgr fail", "who": "smithi196.voprex"}]': finished 2024-02-19T21:54:21.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:20.491704+0000 mon.smithi131 (mon.0) 896 : cluster [DBG] mgrmap e28: smithi131.cjubsp(active, starting, since 0.995722s) 2024-02-19T21:54:21.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.493252+0000 mon.smithi131 (mon.0) 897 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:54:21.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.493464+0000 mon.smithi131 (mon.0) 898 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.493833+0000 mon.smithi131 (mon.0) 899 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.494117+0000 mon.smithi131 (mon.0) 900 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.494393+0000 mon.smithi131 (mon.0) 901 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi131.cjubsp", "id": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.494626+0000 mon.smithi131 (mon.0) 902 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.494871+0000 mon.smithi131 (mon.0) 903 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.495102+0000 mon.smithi131 (mon.0) 904 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.495350+0000 mon.smithi131 (mon.0) 905 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.495656+0000 mon.smithi131 (mon.0) 906 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.495887+0000 mon.smithi131 (mon.0) 907 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.496157+0000 mon.smithi131 (mon.0) 908 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.496422+0000 mon.smithi131 (mon.0) 909 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.496683+0000 mon.smithi131 (mon.0) 910 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-19T21:54:21.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.496958+0000 mon.smithi131 (mon.0) 911 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-19T21:54:21.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.497383+0000 mon.smithi131 (mon.0) 912 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-19T21:54:21.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:20.524925+0000 mon.smithi131 (mon.0) 913 : cluster [INF] Manager daemon smithi131.cjubsp is now available 2024-02-19T21:54:21.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.629772+0000 mon.smithi131 (mon.0) 914 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:21.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.631219+0000 mon.smithi131 (mon.0) 915 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:21.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.681554+0000 mon.smithi131 (mon.0) 916 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:54:21.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:20.686709+0000 mon.smithi131 (mon.0) 917 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.491654+0000 mon.smithi131 (mon.0) 895 : audit [INF] from='mgr.14614 172.21.15.196:0/1424213189' entity='mgr.smithi196.voprex' cmd='[{"prefix": "mgr fail", "who": "smithi196.voprex"}]': finished 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:20.491704+0000 mon.smithi131 (mon.0) 896 : cluster [DBG] mgrmap e28: smithi131.cjubsp(active, starting, since 0.995722s) 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.493252+0000 mon.smithi131 (mon.0) 897 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.493464+0000 mon.smithi131 (mon.0) 898 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.493833+0000 mon.smithi131 (mon.0) 899 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.494117+0000 mon.smithi131 (mon.0) 900 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.494393+0000 mon.smithi131 (mon.0) 901 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi131.cjubsp", "id": "smithi131.cjubsp"}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.494626+0000 mon.smithi131 (mon.0) 902 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.494871+0000 mon.smithi131 (mon.0) 903 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.495102+0000 mon.smithi131 (mon.0) 904 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:54:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.495350+0000 mon.smithi131 (mon.0) 905 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.495656+0000 mon.smithi131 (mon.0) 906 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.495887+0000 mon.smithi131 (mon.0) 907 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.496157+0000 mon.smithi131 (mon.0) 908 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.496422+0000 mon.smithi131 (mon.0) 909 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.496683+0000 mon.smithi131 (mon.0) 910 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.496958+0000 mon.smithi131 (mon.0) 911 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.497383+0000 mon.smithi131 (mon.0) 912 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:20.524925+0000 mon.smithi131 (mon.0) 913 : cluster [INF] Manager daemon smithi131.cjubsp is now available 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.629772+0000 mon.smithi131 (mon.0) 914 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.631219+0000 mon.smithi131 (mon.0) 915 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.681554+0000 mon.smithi131 (mon.0) 916 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/mirror_snapshot_schedule"}]: dispatch 2024-02-19T21:54:21.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:20.686709+0000 mon.smithi131 (mon.0) 917 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi131.cjubsp/trash_purge_schedule"}]: dispatch 2024-02-19T21:54:22.637 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:21.497609+0000 mon.smithi131 (mon.0) 918 : cluster [DBG] mgrmap e29: smithi131.cjubsp(active, since 2s) 2024-02-19T21:54:22.637 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:21.507204+0000 mgr.smithi131.cjubsp (mgr.14720) 1 : cluster [DBG] pgmap v3: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:54:22.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:21.497609+0000 mon.smithi131 (mon.0) 918 : cluster [DBG] mgrmap e29: smithi131.cjubsp(active, since 2s) 2024-02-19T21:54:22.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:21.507204+0000 mgr.smithi131.cjubsp (mgr.14720) 1 : cluster [DBG] pgmap v3: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:54:23.570 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:22.495689+0000 mgr.smithi131.cjubsp (mgr.14720) 2 : cluster [DBG] pgmap v4: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:54:23.571 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:22.500991+0000 mon.smithi131 (mon.0) 919 : cluster [DBG] mgrmap e30: smithi131.cjubsp(active, since 3s) 2024-02-19T21:54:23.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:22.495689+0000 mgr.smithi131.cjubsp (mgr.14720) 2 : cluster [DBG] pgmap v4: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:54:23.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:22.500991+0000 mon.smithi131 (mon.0) 919 : cluster [DBG] mgrmap e30: smithi131.cjubsp(active, since 3s) 2024-02-19T21:54:24.741 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:25.903 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:24.496146+0000 mgr.smithi131.cjubsp (mgr.14720) 3 : cluster [DBG] pgmap v5: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:54:25.904 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:25.515442+0000 mon.smithi131 (mon.0) 920 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:26.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:24.496146+0000 mgr.smithi131.cjubsp (mgr.14720) 3 : cluster [DBG] pgmap v5: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:54:26.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:25.515442+0000 mon.smithi131 (mon.0) 920 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:26.166866+0000 mon.smithi131 (mon.0) 921 : cluster [DBG] Standby manager daemon smithi196.voprex started 2024-02-19T21:54:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:26.168130+0000 mon.smithi196 (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.196:0/4249876967' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/crt"}]: dispatch 2024-02-19T21:54:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:26.168684+0000 mon.smithi196 (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.196:0/4249876967' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:54:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:26.170919+0000 mon.smithi196 (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.196:0/4249876967' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/key"}]: dispatch 2024-02-19T21:54:27.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:26.171824+0000 mon.smithi196 (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.196:0/4249876967' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:54:27.198 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:26.166866+0000 mon.smithi131 (mon.0) 921 : cluster [DBG] Standby manager daemon smithi196.voprex started 2024-02-19T21:54:27.198 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:26.168130+0000 mon.smithi196 (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.196:0/4249876967' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/crt"}]: dispatch 2024-02-19T21:54:27.198 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:26.168684+0000 mon.smithi196 (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.196:0/4249876967' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:54:27.198 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:26.170919+0000 mon.smithi196 (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.196:0/4249876967' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/key"}]: dispatch 2024-02-19T21:54:27.198 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:26.171824+0000 mon.smithi196 (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.196:0/4249876967' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:54:28.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:26.496542+0000 mgr.smithi131.cjubsp (mgr.14720) 4 : cluster [DBG] pgmap v6: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:54:28.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:26.851643+0000 mon.smithi131 (mon.0) 922 : cluster [DBG] mgrmap e31: smithi131.cjubsp(active, since 7s), standbys: smithi196.voprex 2024-02-19T21:54:28.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:26.854299+0000 mon.smithi131 (mon.0) 923 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi196.voprex", "id": "smithi196.voprex"}]: dispatch 2024-02-19T21:54:28.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:27.616424+0000 mon.smithi131 (mon.0) 924 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:26.496542+0000 mgr.smithi131.cjubsp (mgr.14720) 4 : cluster [DBG] pgmap v6: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-02-19T21:54:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:26.851643+0000 mon.smithi131 (mon.0) 922 : cluster [DBG] mgrmap e31: smithi131.cjubsp(active, since 7s), standbys: smithi196.voprex 2024-02-19T21:54:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:26.854299+0000 mon.smithi131 (mon.0) 923 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr metadata", "who": "smithi196.voprex", "id": "smithi196.voprex"}]: dispatch 2024-02-19T21:54:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:27.616424+0000 mon.smithi131 (mon.0) 924 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:27.897767+0000 mon.smithi131 (mon.0) 925 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:27.898499+0000 mon.smithi131 (mon.0) 926 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:27.899532+0000 mgr.smithi131.cjubsp (mgr.14720) 5 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.conf 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:27.899731+0000 mgr.smithi131.cjubsp (mgr.14720) 6 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.conf 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:27.925881+0000 mgr.smithi131.cjubsp (mgr.14720) 7 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:27.926888+0000 mgr.smithi131.cjubsp (mgr.14720) 8 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:27.956558+0000 mon.smithi131 (mon.0) 927 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:27.959958+0000 mon.smithi131 (mon.0) 928 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:27.965012+0000 mon.smithi131 (mon.0) 929 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.165 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:27.969625+0000 mon.smithi131 (mon.0) 930 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.166 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:27.972452+0000 mon.smithi131 (mon.0) 931 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:27.897767+0000 mon.smithi131 (mon.0) 925 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:27.898499+0000 mon.smithi131 (mon.0) 926 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:27.899532+0000 mgr.smithi131.cjubsp (mgr.14720) 5 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.conf 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:27.899731+0000 mgr.smithi131.cjubsp (mgr.14720) 6 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.conf 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:27.925881+0000 mgr.smithi131.cjubsp (mgr.14720) 7 : cephadm [INF] Updating smithi131:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:27.926888+0000 mgr.smithi131.cjubsp (mgr.14720) 8 : cephadm [INF] Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:27.956558+0000 mon.smithi131 (mon.0) 927 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:27.959958+0000 mon.smithi131 (mon.0) 928 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:27.965012+0000 mon.smithi131 (mon.0) 929 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:27.969625+0000 mon.smithi131 (mon.0) 930 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:29.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:27.972452+0000 mon.smithi131 (mon.0) 931 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:29.790 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:28.497213+0000 mgr.smithi131.cjubsp (mgr.14720) 9 : cluster [DBG] pgmap v7: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 292 B/s rd, 146 B/s wr, 0 op/s 2024-02-19T21:54:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:29.611235+0000 mon.smithi131 (mon.0) 932 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi196.voprex", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:54:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:29.612191+0000 mon.smithi131 (mon.0) 933 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:54:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:29.613056+0000 mon.smithi131 (mon.0) 934 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:30.330 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:28.497213+0000 mgr.smithi131.cjubsp (mgr.14720) 9 : cluster [DBG] pgmap v7: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 292 B/s rd, 146 B/s wr, 0 op/s 2024-02-19T21:54:30.330 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:29.611235+0000 mon.smithi131 (mon.0) 932 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi196.voprex", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-19T21:54:30.330 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:29.612191+0000 mon.smithi131 (mon.0) 933 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-19T21:54:30.330 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:29.613056+0000 mon.smithi131 (mon.0) 934 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:31.127 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:29.610841+0000 mgr.smithi131.cjubsp (mgr.14720) 10 : cephadm [INF] Upgrade: Updating mgr.smithi196.voprex 2024-02-19T21:54:31.128 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:29.613765+0000 mgr.smithi131.cjubsp (mgr.14720) 11 : cephadm [INF] Deploying daemon mgr.smithi196.voprex on smithi196 2024-02-19T21:54:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:29.610841+0000 mgr.smithi131.cjubsp (mgr.14720) 10 : cephadm [INF] Upgrade: Updating mgr.smithi196.voprex 2024-02-19T21:54:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:29.613765+0000 mgr.smithi131.cjubsp (mgr.14720) 11 : cephadm [INF] Deploying daemon mgr.smithi196.voprex on smithi196 2024-02-19T21:54:32.104 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:30.497708+0000 mgr.smithi131.cjubsp (mgr.14720) 12 : cluster [DBG] pgmap v8: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 227 B/s rd, 113 B/s wr, 0 op/s 2024-02-19T21:54:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:30.497708+0000 mgr.smithi131.cjubsp (mgr.14720) 12 : cluster [DBG] pgmap v8: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 227 B/s rd, 113 B/s wr, 0 op/s 2024-02-19T21:54:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:32.498540+0000 mgr.smithi131.cjubsp (mgr.14720) 13 : cluster [DBG] pgmap v9: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 186 B/s wr, 0 op/s 2024-02-19T21:54:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:33.128002+0000 mon.smithi131 (mon.0) 935 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:33.128865+0000 mon.smithi131 (mon.0) 936 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:32.498540+0000 mgr.smithi131.cjubsp (mgr.14720) 13 : cluster [DBG] pgmap v9: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 186 B/s wr, 0 op/s 2024-02-19T21:54:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:33.128002+0000 mon.smithi131 (mon.0) 935 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:33.128865+0000 mon.smithi131 (mon.0) 936 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:34.838 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:34.499188+0000 mgr.smithi131.cjubsp (mgr.14720) 14 : cluster [DBG] pgmap v10: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:34.499188+0000 mgr.smithi131.cjubsp (mgr.14720) 14 : cluster [DBG] pgmap v10: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:38.379 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:36.499702+0000 mgr.smithi131.cjubsp (mgr.14720) 15 : cluster [DBG] pgmap v11: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:38.379 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.776506+0000 mon.smithi131 (mon.0) 937 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:38.379 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.777453+0000 mon.smithi131 (mon.0) 938 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:38.379 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.778405+0000 mon.smithi131 (mon.0) 939 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.783173+0000 mon.smithi131 (mon.0) 940 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.789386+0000 mon.smithi131 (mon.0) 941 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.792639+0000 mon.smithi131 (mon.0) 942 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.794913+0000 mon.smithi131 (mon.0) 943 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.799475+0000 mon.smithi131 (mon.0) 944 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.800316+0000 mon.smithi131 (mon.0) 945 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi131.cjubsp"}]: dispatch 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.803723+0000 mon.smithi131 (mon.0) 946 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi131.cjubsp"}]': finished 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.804347+0000 mon.smithi131 (mon.0) 947 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi196.voprex"}]: dispatch 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.807706+0000 mon.smithi131 (mon.0) 948 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi196.voprex"}]': finished 2024-02-19T21:54:38.380 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:37.808747+0000 mon.smithi131 (mon.0) 949 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:36.499702+0000 mgr.smithi131.cjubsp (mgr.14720) 15 : cluster [DBG] pgmap v11: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.776506+0000 mon.smithi131 (mon.0) 937 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.777453+0000 mon.smithi131 (mon.0) 938 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.778405+0000 mon.smithi131 (mon.0) 939 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.783173+0000 mon.smithi131 (mon.0) 940 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.789386+0000 mon.smithi131 (mon.0) 941 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.792639+0000 mon.smithi131 (mon.0) 942 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.794913+0000 mon.smithi131 (mon.0) 943 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:54:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.799475+0000 mon.smithi131 (mon.0) 944 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:38.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.800316+0000 mon.smithi131 (mon.0) 945 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi131.cjubsp"}]: dispatch 2024-02-19T21:54:38.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.803723+0000 mon.smithi131 (mon.0) 946 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi131.cjubsp"}]': finished 2024-02-19T21:54:38.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.804347+0000 mon.smithi131 (mon.0) 947 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi196.voprex"}]: dispatch 2024-02-19T21:54:38.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.807706+0000 mon.smithi131 (mon.0) 948 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi196.voprex"}]': finished 2024-02-19T21:54:38.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:37.808747+0000 mon.smithi131 (mon.0) 949 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-19T21:54:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:37.795532+0000 mgr.smithi131.cjubsp (mgr.14720) 16 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-02-19T21:54:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:38.654911+0000 mon.smithi131 (mon.0) 950 : cluster [DBG] Standby manager daemon smithi196.voprex restarted 2024-02-19T21:54:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:38.655021+0000 mon.smithi131 (mon.0) 951 : cluster [DBG] Standby manager daemon smithi196.voprex started 2024-02-19T21:54:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:38.656141+0000 mon.smithi196 (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.196:0/758238431' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/crt"}]: dispatch 2024-02-19T21:54:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:38.656618+0000 mon.smithi196 (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.196:0/758238431' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:54:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:38.658519+0000 mon.smithi196 (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.196:0/758238431' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/key"}]: dispatch 2024-02-19T21:54:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:38.659525+0000 mon.smithi196 (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.196:0/758238431' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:54:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cephadm 2024-02-19T21:54:37.795532+0000 mgr.smithi131.cjubsp (mgr.14720) 16 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-02-19T21:54:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:38.654911+0000 mon.smithi131 (mon.0) 950 : cluster [DBG] Standby manager daemon smithi196.voprex restarted 2024-02-19T21:54:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:38.655021+0000 mon.smithi131 (mon.0) 951 : cluster [DBG] Standby manager daemon smithi196.voprex started 2024-02-19T21:54:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:38.656141+0000 mon.smithi196 (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.196:0/758238431' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/crt"}]: dispatch 2024-02-19T21:54:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:38.656618+0000 mon.smithi196 (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.196:0/758238431' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-19T21:54:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:38.658519+0000 mon.smithi196 (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.196:0/758238431' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi196.voprex/key"}]: dispatch 2024-02-19T21:54:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:38.659525+0000 mon.smithi196 (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.196:0/758238431' entity='mgr.smithi196.voprex' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-19T21:54:39.893 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:38.500342+0000 mgr.smithi131.cjubsp (mgr.14720) 17 : cluster [DBG] pgmap v12: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: cluster 2024-02-19T21:54:39.145116+0000 mon.smithi131 (mon.0) 952 : cluster [DBG] mgrmap e32: smithi131.cjubsp(active, since 19s), standbys: smithi196.voprex 2024-02-19T21:54:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:39.279564+0000 mon.smithi131 (mon.0) 953 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:39.280175+0000 mon.smithi131 (mon.0) 954 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:54:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:39.280642+0000 mon.smithi131 (mon.0) 955 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:54:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: audit 2024-02-19T21:54:39.281115+0000 mon.smithi131 (mon.0) 956 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:38.500342+0000 mgr.smithi131.cjubsp (mgr.14720) 17 : cluster [DBG] pgmap v12: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:39.145116+0000 mon.smithi131 (mon.0) 952 : cluster [DBG] mgrmap e32: smithi131.cjubsp(active, since 19s), standbys: smithi196.voprex 2024-02-19T21:54:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:39.279564+0000 mon.smithi131 (mon.0) 953 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:39.280175+0000 mon.smithi131 (mon.0) 954 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:54:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:39.280642+0000 mon.smithi131 (mon.0) 955 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:54:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:39.281115+0000 mon.smithi131 (mon.0) 956 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:40.844 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 systemd[1]: Stopping Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:54:41.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:54:40.843+0000 7f82a69fa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi131 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-02-19T21:54:41.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131[34430]: debug 2024-02-19T21:54:40.843+0000 7f82a69fa700 -1 mon.smithi131@0(leader) e2 *** Got Signal Terminated *** 2024-02-19T21:54:41.661 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:41 smithi131 bash[76559]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131 2024-02-19T21:54:41.922 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:41 smithi131 bash[76622]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131 2024-02-19T21:54:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131.service: Succeeded. 2024-02-19T21:54:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 systemd[1]: Stopped Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:54:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 systemd[1]: Starting Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:54:43.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.762+0000 7fbb211db880 0 set uid:gid to 167:167 (ceph:ceph) 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.762+0000 7fbb211db880 0 ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable), process ceph-mon, pid 7 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.762+0000 7fbb211db880 0 pidfile_write: ignore empty --pid-file 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.766+0000 7fbb211db880 0 load: jerasure load: lrc load: isa 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: RocksDB version: 6.8.1 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Compile date Feb 17 2024 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: DB SUMMARY 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: CURRENT file: CURRENT 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: IDENTITY file: IDENTITY 2024-02-19T21:54:43.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 766 Bytes 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi131/store.db dir, Total Num: 1, files: 000020.sst 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi131/store.db: 000018.log size: 3880291 ; 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.error_if_exists: 0 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.create_if_missing: 0 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.paranoid_checks: 1 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.env: 0x55fc64d2a080 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.fs: Posix File System 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.info_log: 0x55fc66078560 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_file_opening_threads: 16 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.statistics: (nil) 2024-02-19T21:54:43.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.use_fsync: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_log_file_size: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.log_file_time_to_roll: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.keep_log_file_num: 1000 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.recycle_log_file_num: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.allow_fallocate: 1 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.allow_mmap_reads: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.allow_mmap_writes: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.use_direct_reads: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.create_missing_column_families: 0 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.db_log_dir: 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi131/store.db 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.table_cache_numshardbits: 6 2024-02-19T21:54:43.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_subcompactions: 1 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_background_flushes: -1 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.advise_random_on_open: 1 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.db_write_buffer_size: 0 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.write_buffer_manager: 0x55fc66d6a3f0 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.use_adaptive_mutex: 0 2024-02-19T21:54:43.260 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.rate_limiter: (nil) 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.wal_recovery_mode: 2 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.enable_thread_tracking: 0 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.enable_pipelined_write: 0 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.unordered_write: 0 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.row_cache: None 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.wal_filter: None 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.allow_ingest_behind: 0 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.preserve_deletes: 0 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.two_write_queues: 0 2024-02-19T21:54:43.261 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.manual_wal_flush: 0 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.atomic_flush: 0 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.persist_stats_to_disk: 0 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.log_readahead_size: 0 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_background_jobs: 2 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_background_compactions: -1 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.delayed_write_rate : 16777216 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_total_wal_size: 0 2024-02-19T21:54:43.262 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.stats_dump_period_sec: 600 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.stats_persist_period_sec: 600 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.max_open_files: -1 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.bytes_per_sync: 0 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Options.compaction_readahead_size: 0 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Compression algorithms supported: 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: kZSTD supported: 0 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: kXpressCompression supported: 0 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: kLZ4HCCompression supported: 1 2024-02-19T21:54:43.263 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: kLZ4Compression supported: 1 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: kBZip2Compression supported: 0 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: kZlibCompression supported: 1 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: kSnappyCompression supported: 1 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.767+0000 7fbb211db880 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi131/store.db/MANIFEST-000008 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.merge_operator: 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_filter: None 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_filter_factory: None 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-02-19T21:54:43.264 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.table_factory: BlockBasedTable 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fc6601dd10) 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cache_index_and_filter_blocks: 1 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: pin_top_level_index_and_filter: 1 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: index_type: 0 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: data_block_index_type: 0 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: index_shortening: 1 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: data_block_hash_table_util_ratio: 0.750000 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: hash_index_allow_collision: 1 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: checksum: 1 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: no_block_cache: 0 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: block_cache: 0x55fc66052f10 2024-02-19T21:54:43.265 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: block_cache_name: BinnedLRUCache 2024-02-19T21:54:43.266 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: block_cache_options: 2024-02-19T21:54:43.266 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: capacity : 536870912 2024-02-19T21:54:43.266 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: num_shard_bits : 4 2024-02-19T21:54:43.266 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: strict_capacity_limit : 0 2024-02-19T21:54:43.266 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: high_pri_pool_ratio: 0.000 2024-02-19T21:54:43.267 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: block_cache_compressed: (nil) 2024-02-19T21:54:43.267 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: persistent_cache: (nil) 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: block_size: 4096 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: block_size_deviation: 10 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: block_restart_interval: 16 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: index_block_restart_interval: 1 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: metadata_block_size: 4096 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: partition_filters: 0 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: use_delta_encoding: 1 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: whole_key_filtering: 1 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: verify_compression: 0 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: read_amp_bytes_per_bit: 0 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: format_version: 2 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: enable_index_compression: 1 2024-02-19T21:54:43.268 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: block_align: 0 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.write_buffer_size: 33554432 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_write_buffer_number: 2 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compression: NoCompression 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.bottommost_compression: Disabled 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.prefix_extractor: nullptr 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.num_levels: 7 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-19T21:54:43.269 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compression_opts.window_bits: -14 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compression_opts.level: 32767 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compression_opts.strategy: 0 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compression_opts.enabled: false 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.target_file_size_base: 67108864 2024-02-19T21:54:43.270 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.target_file_size_multiplier: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-19T21:54:43.271 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.arena_block_size: 4194304 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.disable_auto_compactions: 0 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-19T21:54:43.272 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.table_properties_collectors: 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.inplace_update_support: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.memtable_huge_page_size: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.bloom_locality: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.max_successive_merges: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.paranoid_file_checks: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.force_consistency_checks: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.report_bg_io_stats: 0 2024-02-19T21:54:43.273 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.ttl: 2592000 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: [version_set.cc:3291] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi131/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 22, last_sequence is 9790, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 18 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379682769642, "job": 1, "event": "recovery_started", "log_files": [18]} 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.768+0000 7fbb211db880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #18 mode 2 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.772+0000 7fbb211db880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.778+0000 7fbb211db880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379682779013, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 22, "file_size": 3762034, "table_properties": {"data_size": 3756014, "index_size": 4492, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 709, "raw_key_size": 5614, "raw_average_key_size": 22, "raw_value_size": 3748681, "raw_average_value_size": 15054, "num_data_blocks": 143, "num_entries": 249, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1708379682, "oldest_key_time": 3, "file_creation_time": 0}} 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.778+0000 7fbb211db880 4 rocksdb: [version_set.cc:3826] Creating manifest 23 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.778+0000 7fbb211db880 4 rocksdb: [version_set.cc:3291] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.780+0000 7fbb211db880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379682781022, "job": 1, "event": "recovery_finished"} 2024-02-19T21:54:43.274 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.781+0000 7fbb211db880 4 rocksdb: DB pointer 0x55fc6611b800 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.781+0000 7fbb07fcd700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.781+0000 7fbb07fcd700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ** DB Stats ** 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ** Compaction Stats [default] ** 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: L0 1/0 3.59 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 564.6 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:43.275 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: L6 1/0 23.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Sum 2/0 27.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 564.6 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 564.6 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ** Compaction Stats [default] ** 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 564.6 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Flush(GB): cumulative 0.004, interval 0.004 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:54:43.276 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Cumulative compaction: 0.00 GB write, 270.06 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Interval compaction: 0.00 GB write, 270.06 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ** Compaction Stats [default] ** 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: L0 1/0 3.59 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 564.6 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: L6 1/0 23.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:54:43.277 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Sum 2/0 27.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 564.6 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ** Compaction Stats [default] ** 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 564.6 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Flush(GB): cumulative 0.004, interval 0.000 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Cumulative compaction: 0.00 GB write, 269.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:54:43.278 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.781+0000 7fbb211db880 0 starting mon.smithi131 rank 0 at public addrs [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] at bind addrs [v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi131 fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.782+0000 7fbb211db880 1 mon.smithi131@-1(???) e2 preinit fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.783+0000 7fbb211db880 0 mon.smithi131@-1(???).mds e7 new map 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.783+0000 7fbb211db880 0 mon.smithi131@-1(???).mds e7 print_map 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: e7 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: enable_multiple, ever_enabled_multiple: 1,1 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: legacy client fscid: 1 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Filesystem 'foofs' (1) 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: fs_name foofs 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: epoch 4 2024-02-19T21:54:43.279 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: flags 12 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: created 2024-02-19T21:51:14.830583+0000 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: modified 2024-02-19T21:51:19.787557+0000 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: tableserver 0 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: root 0 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: session_timeout 60 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: session_autoclose 300 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: max_file_size 1099511627776 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: required_client_features {} 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: last_failure 0 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: last_failure_osd_epoch 0 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: max_mds 1 2024-02-19T21:54:43.280 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: in 0 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: up {0=14474} 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: failed 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: damaged 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: stopped 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: data_pools [3] 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: metadata_pool 2 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: inline_data disabled 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: balancer 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: standby_count_wanted 1 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: [mds.foofs.smithi131.gemwnz{0:14474} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639] compat {c=[1],r=[1],i=[7ff]}] 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: Standby daemons: 2024-02-19T21:54:43.281 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: [mds.foofs.smithi196.hhhkwt{-1:24311} state up:standby seq 3 join_fscid=1 addr [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] compat {c=[1],r=[1],i=[7ff]}] 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.783+0000 7fbb211db880 0 mon.smithi131@-1(???).osd e70 crush map has features 3314933000854323200, adjusting msgr requires 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.783+0000 7fbb211db880 0 mon.smithi131@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.783+0000 7fbb211db880 0 mon.smithi131@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.783+0000 7fbb211db880 0 mon.smithi131@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:38.500342+0000 mgr.smithi131.cjubsp (mgr.14720) 17 : cluster [DBG] pgmap v12: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:39.145116+0000 mon.smithi131 (mon.0) 952 : cluster [DBG] mgrmap e32: smithi131.cjubsp(active, since 19s), standbys: smithi196.voprex 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:39.279564+0000 mon.smithi131 (mon.0) 953 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:39.280175+0000 mon.smithi131 (mon.0) 954 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:39.280642+0000 mon.smithi131 (mon.0) 955 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:39.281115+0000 mon.smithi131 (mon.0) 956 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:54:42.783+0000 7fbb211db880 1 mon.smithi131@-1(???).paxosservice(auth 1..42) refresh upgraded, format 0 -> 3 2024-02-19T21:54:43.282 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 bash[76767]: 83bc30cf7493b4f94ed166cd15594ec8f2bf90efffc4d5861eaa315b6e4227b4 2024-02-19T21:54:43.283 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:42 smithi131 systemd[1]: Started Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:54:44.097 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:54:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:40.852910+0000 mon.smithi196 (mon.1) 27 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:54:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:40.853484+0000 mon.smithi196 (mon.1) 28 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:54:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:42.501636+0000 mgr.smithi131.cjubsp (mgr.14720) 21 : cluster [DBG] pgmap v14: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:44.084104+0000 mgr.smithi131.cjubsp (mgr.14720) 22 : audit [DBG] from='client.24575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.195067+0000 mon.smithi131 (mon.0) 1 : cluster [INF] mon.smithi131 calling monitor election 2024-02-19T21:54:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.208188+0000 mon.smithi131 (mon.0) 2 : cluster [INF] mon.smithi131 is new leader, mons smithi131,smithi196 in quorum (ranks 0,1) 2024-02-19T21:54:44.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.212379+0000 mon.smithi131 (mon.0) 3 : cluster [DBG] monmap e2: 2 mons at {smithi131=[v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0],smithi196=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} removed_ranks: {} 2024-02-19T21:54:44.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.212879+0000 mon.smithi131 (mon.0) 4 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:54:44.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.212923+0000 mon.smithi131 (mon.0) 5 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-02-19T21:54:44.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.213864+0000 mon.smithi131 (mon.0) 6 : cluster [DBG] mgrmap e32: smithi131.cjubsp(active, since 24s), standbys: smithi196.voprex 2024-02-19T21:54:44.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.214091+0000 mon.smithi131 (mon.0) 7 : cluster [INF] overall HEALTH_OK 2024-02-19T21:54:44.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:44.220482+0000 mon.smithi131 (mon.0) 8 : audit [INF] from='mgr.14720 ' entity='' 2024-02-19T21:54:44.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:44.222464+0000 mon.smithi196 (mon.1) 29 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:44.713 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:54:44.713 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (7m) 17s ago 9m 21.2M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:54:44.713 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (9m) 17s ago 9m 6387k - 16.2.5 6933c2a0b7dd 7e4833221591 2024-02-19T21:54:44.713 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (8m) 6s ago 8m 6395k - 16.2.5 6933c2a0b7dd d566245c5426 2024-02-19T21:54:44.713 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (7m) 17s ago 8m 34.4M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (3m) 17s ago 3m 38.5M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (3m) 6s ago 3m 13.0M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (32s) 17s ago 10m 413M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (12s) 6s ago 8m 43.8M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 starting - - - 2048M 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (8m) 6s ago 8m 42.7M 2048M 16.2.5 6933c2a0b7dd 6a272417d66d 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (66s) 17s ago 65s 50.7M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (59s) 6s ago 58s 49.2M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (8m) 17s ago 8m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (7m) 6s ago 7m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (7m) 17s ago 7m 373M 1944M 16.2.5 6933c2a0b7dd f42dfded3a0d 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (7m) 17s ago 7m 286M 1944M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (6m) 17s ago 6m 226M 1944M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:54:44.714 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (6m) 17s ago 6m 340M 1944M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:54:44.715 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (6m) 6s ago 6m 355M 2712M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:54:44.715 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (5m) 6s ago 5m 374M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:54:44.715 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (5m) 6s ago 5m 311M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:54:44.715 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (5m) 6s ago 5m 467M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:54:44.715 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (37s) 17s ago 8m 45.0M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:40.852910+0000 mon.smithi196 (mon.1) 27 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:40.853484+0000 mon.smithi196 (mon.1) 28 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:42.501636+0000 mgr.smithi131.cjubsp (mgr.14720) 21 : cluster [DBG] pgmap v14: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:44.084104+0000 mgr.smithi131.cjubsp (mgr.14720) 22 : audit [DBG] from='client.24575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.195067+0000 mon.smithi131 (mon.0) 1 : cluster [INF] mon.smithi131 calling monitor election 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.208188+0000 mon.smithi131 (mon.0) 2 : cluster [INF] mon.smithi131 is new leader, mons smithi131,smithi196 in quorum (ranks 0,1) 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.212379+0000 mon.smithi131 (mon.0) 3 : cluster [DBG] monmap e2: 2 mons at {smithi131=[v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0],smithi196=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} removed_ranks: {} 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.212879+0000 mon.smithi131 (mon.0) 4 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.212923+0000 mon.smithi131 (mon.0) 5 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.213864+0000 mon.smithi131 (mon.0) 6 : cluster [DBG] mgrmap e32: smithi131.cjubsp(active, since 24s), standbys: smithi196.voprex 2024-02-19T21:54:44.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.214091+0000 mon.smithi131 (mon.0) 7 : cluster [INF] overall HEALTH_OK 2024-02-19T21:54:44.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:44.220482+0000 mon.smithi131 (mon.0) 8 : audit [INF] from='mgr.14720 ' entity='' 2024-02-19T21:54:44.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:44.222464+0000 mon.smithi196 (mon.1) 29 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:44.935 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 1, 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:54:45.070 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 3, 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 11 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:54:45.071 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:54:45.372 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "2/23 daemons upgraded", 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading mon daemons", 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:54:45.373 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:54:45.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.268566+0000 mon.smithi131 (mon.0) 9 : cluster [DBG] mgrmap e33: smithi131.cjubsp(active, since 24s), standbys: smithi196.voprex 2024-02-19T21:54:45.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:44.389925+0000 mgr.smithi131.cjubsp (mgr.14720) 23 : audit [DBG] from='client.24579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:45.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:44.502123+0000 mgr.smithi131.cjubsp (mgr.14720) 24 : cluster [DBG] pgmap v15: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:45.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:44.705337+0000 mgr.smithi131.cjubsp (mgr.14720) 25 : audit [DBG] from='client.34100 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:45.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:45.070876+0000 mon.smithi131 (mon.0) 10 : audit [DBG] from='client.? 172.21.15.131:0/131167430' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:54:45.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.268566+0000 mon.smithi131 (mon.0) 9 : cluster [DBG] mgrmap e33: smithi131.cjubsp(active, since 24s), standbys: smithi196.voprex 2024-02-19T21:54:45.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:44.389925+0000 mgr.smithi131.cjubsp (mgr.14720) 23 : audit [DBG] from='client.24579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:45.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:44.502123+0000 mgr.smithi131.cjubsp (mgr.14720) 24 : cluster [DBG] pgmap v15: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:45.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:44.705337+0000 mgr.smithi131.cjubsp (mgr.14720) 25 : audit [DBG] from='client.34100 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:45.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:45.070876+0000 mon.smithi131 (mon.0) 10 : audit [DBG] from='client.? 172.21.15.131:0/131167430' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:54:46.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:45.373694+0000 mgr.smithi131.cjubsp (mgr.14720) 26 : audit [DBG] from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:46.751 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:45.373694+0000 mgr.smithi131.cjubsp (mgr.14720) 26 : audit [DBG] from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:54:47.566 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:46.502657+0000 mgr.smithi131.cjubsp (mgr.14720) 27 : cluster [DBG] pgmap v16: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:47.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:46.502657+0000 mgr.smithi131.cjubsp (mgr.14720) 27 : cluster [DBG] pgmap v16: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:49.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:48.503486+0000 mgr.smithi131.cjubsp (mgr.14720) 28 : cluster [DBG] pgmap v17: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:49.977 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:50.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:48.503486+0000 mgr.smithi131.cjubsp (mgr.14720) 28 : cluster [DBG] pgmap v17: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:50.503915+0000 mgr.smithi131.cjubsp (mgr.14720) 29 : cluster [DBG] pgmap v18: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:50.995912+0000 mon.smithi131 (mon.0) 11 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:50.999642+0000 mon.smithi196 (mon.1) 30 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:51.000430+0000 mon.smithi196 (mon.1) 31 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:54:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:51.004742+0000 mon.smithi131 (mon.0) 12 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:51.011001+0000 mon.smithi131 (mon.0) 13 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:51.014312+0000 mon.smithi196 (mon.1) 32 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:51.017237+0000 mon.smithi196 (mon.1) 33 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:54:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:51.018543+0000 mon.smithi196 (mon.1) 34 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-19T21:54:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:50.503915+0000 mgr.smithi131.cjubsp (mgr.14720) 29 : cluster [DBG] pgmap v18: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:50.995912+0000 mon.smithi131 (mon.0) 11 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:50.999642+0000 mon.smithi196 (mon.1) 30 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:51.000430+0000 mon.smithi196 (mon.1) 31 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:54:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:51.004742+0000 mon.smithi131 (mon.0) 12 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:51.011001+0000 mon.smithi131 (mon.0) 13 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:51.014312+0000 mon.smithi196 (mon.1) 32 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:52.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:51.017237+0000 mon.smithi196 (mon.1) 33 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:54:52.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:51.018543+0000 mon.smithi196 (mon.1) 34 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-19T21:54:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cluster 2024-02-19T21:54:52.504658+0000 mgr.smithi131.cjubsp (mgr.14720) 30 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:52.615477+0000 mgr.smithi131.cjubsp (mgr.14720) 31 : cephadm [INF] Upgrade: Updating mon.smithi196 2024-02-19T21:54:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:52.620230+0000 mon.smithi131 (mon.0) 14 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:52.621224+0000 mon.smithi196 (mon.1) 35 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:54:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:52.621911+0000 mon.smithi196 (mon.1) 36 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:54:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: audit 2024-02-19T21:54:52.622563+0000 mon.smithi196 (mon.1) 37 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:53.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: cephadm 2024-02-19T21:54:52.623224+0000 mgr.smithi131.cjubsp (mgr.14720) 32 : cephadm [INF] Deploying daemon mon.smithi196 on smithi196 2024-02-19T21:54:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:52.504658+0000 mgr.smithi131.cjubsp (mgr.14720) 30 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:54:52.615477+0000 mgr.smithi131.cjubsp (mgr.14720) 31 : cephadm [INF] Upgrade: Updating mon.smithi196 2024-02-19T21:54:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:52.620230+0000 mon.smithi131 (mon.0) 14 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:52.621224+0000 mon.smithi196 (mon.1) 35 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:54:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:52.621911+0000 mon.smithi196 (mon.1) 36 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:54:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:52.622563+0000 mon.smithi196 (mon.1) 37 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:54:52.623224+0000 mgr.smithi131.cjubsp (mgr.14720) 32 : cephadm [INF] Deploying daemon mon.smithi196 on smithi196 2024-02-19T21:54:54.389 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:54 smithi196 systemd[1]: Stopping Ceph mon.smithi196 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:54:54.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:54:54.388+0000 7fcf69736700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi196 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-02-19T21:54:54.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196[36646]: debug 2024-02-19T21:54:54.388+0000 7fcf69736700 -1 mon.smithi196@1(peon) e2 *** Got Signal Terminated *** 2024-02-19T21:54:55.024 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:54:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:55 smithi196 bash[57160]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196 2024-02-19T21:54:55.544 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:55 smithi196 bash[57213]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196" found: no such container 2024-02-19T21:54:55.544 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:55 smithi196 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi196.service: Succeeded. 2024-02-19T21:54:55.544 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:55 smithi196 systemd[1]: Stopped Ceph mon.smithi196 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:54:55.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:55 smithi196 systemd[1]: Starting Ceph mon.smithi196 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:54:56.245 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.127+0000 7ff51d6fe880 0 set uid:gid to 167:167 (ceph:ceph) 2024-02-19T21:54:56.245 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.127+0000 7ff51d6fe880 0 ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable), process ceph-mon, pid 7 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.127+0000 7ff51d6fe880 0 pidfile_write: ignore empty --pid-file 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.131+0000 7ff51d6fe880 0 load: jerasure load: lrc load: isa 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: RocksDB version: 6.8.1 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Compile date Feb 17 2024 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: DB SUMMARY 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: CURRENT file: CURRENT 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: IDENTITY file: IDENTITY 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 690 Bytes 2024-02-19T21:54:56.246 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi196/store.db dir, Total Num: 1, files: 000017.sst 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi196/store.db: 000015.log size: 7099543 ; 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.error_if_exists: 0 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.create_if_missing: 0 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.paranoid_checks: 1 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.env: 0x56322bef9080 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.fs: Posix File System 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.info_log: 0x56322cd36560 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_file_opening_threads: 16 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.statistics: (nil) 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.use_fsync: 0 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_log_file_size: 0 2024-02-19T21:54:56.247 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.log_file_time_to_roll: 0 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.keep_log_file_num: 1000 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.recycle_log_file_num: 0 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.allow_fallocate: 1 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.allow_mmap_reads: 0 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.allow_mmap_writes: 0 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.use_direct_reads: 0 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.create_missing_column_families: 0 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.db_log_dir: 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi196/store.db 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.table_cache_numshardbits: 6 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_subcompactions: 1 2024-02-19T21:54:56.248 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_background_flushes: -1 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.advise_random_on_open: 1 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.db_write_buffer_size: 0 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.write_buffer_manager: 0x56322da283f0 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.use_adaptive_mutex: 0 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.rate_limiter: (nil) 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-19T21:54:56.249 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.wal_recovery_mode: 2 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.enable_thread_tracking: 0 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.enable_pipelined_write: 0 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.unordered_write: 0 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.row_cache: None 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.wal_filter: None 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.allow_ingest_behind: 0 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.preserve_deletes: 0 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.two_write_queues: 0 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.manual_wal_flush: 0 2024-02-19T21:54:56.250 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.atomic_flush: 0 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.persist_stats_to_disk: 0 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.log_readahead_size: 0 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_background_jobs: 2 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_background_compactions: -1 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-19T21:54:56.251 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.delayed_write_rate : 16777216 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_total_wal_size: 0 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.stats_dump_period_sec: 600 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.stats_persist_period_sec: 600 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.max_open_files: -1 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.bytes_per_sync: 0 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.compaction_readahead_size: 0 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Compression algorithms supported: 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: kZSTD supported: 0 2024-02-19T21:54:56.252 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: kXpressCompression supported: 0 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: kLZ4HCCompression supported: 1 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: kLZ4Compression supported: 1 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: kBZip2Compression supported: 0 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: kZlibCompression supported: 1 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: kSnappyCompression supported: 1 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi196/store.db/MANIFEST-000005 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.merge_operator: 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.compaction_filter: None 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.compaction_filter_factory: None 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-02-19T21:54:56.253 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: Options.table_factory: BlockBasedTable 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.132+0000 7ff51d6fe880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56322ccdbd10) 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cache_index_and_filter_blocks: 1 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: pin_top_level_index_and_filter: 1 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: index_type: 0 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: data_block_index_type: 0 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: index_shortening: 1 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: data_block_hash_table_util_ratio: 0.750000 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: hash_index_allow_collision: 1 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: checksum: 1 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: no_block_cache: 0 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: block_cache: 0x56322cd10f10 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: block_cache_name: BinnedLRUCache 2024-02-19T21:54:56.254 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: block_cache_options: 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: capacity : 536870912 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: num_shard_bits : 4 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: strict_capacity_limit : 0 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: high_pri_pool_ratio: 0.000 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: block_cache_compressed: (nil) 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: persistent_cache: (nil) 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: block_size: 4096 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: block_size_deviation: 10 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: block_restart_interval: 16 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: index_block_restart_interval: 1 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: metadata_block_size: 4096 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: partition_filters: 0 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: use_delta_encoding: 1 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-19T21:54:56.255 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: whole_key_filtering: 1 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: verify_compression: 0 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: read_amp_bytes_per_bit: 0 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: format_version: 2 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: enable_index_compression: 1 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: block_align: 0 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.write_buffer_size: 33554432 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_write_buffer_number: 2 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compression: NoCompression 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.bottommost_compression: Disabled 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.prefix_extractor: nullptr 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.num_levels: 7 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-19T21:54:56.256 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compression_opts.window_bits: -14 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compression_opts.level: 32767 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compression_opts.strategy: 0 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compression_opts.enabled: false 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-19T21:54:56.257 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.target_file_size_base: 67108864 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.target_file_size_multiplier: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-19T21:54:56.258 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.arena_block_size: 4194304 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.disable_auto_compactions: 0 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-19T21:54:56.259 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.table_properties_collectors: 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.inplace_update_support: 0 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.memtable_huge_page_size: 0 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.bloom_locality: 0 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.max_successive_merges: 0 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.paranoid_file_checks: 0 2024-02-19T21:54:56.260 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.force_consistency_checks: 0 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.report_bg_io_stats: 0 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.ttl: 2592000 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: [version_set.cc:3291] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi196/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 19, last_sequence is 9888, log_number is 15,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 15 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379696134879, "job": 1, "event": "recovery_started", "log_files": [15]} 2024-02-19T21:54:56.261 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.133+0000 7ff51d6fe880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #15 mode 2 2024-02-19T21:54:56.262 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.141+0000 7ff51d6fe880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-19T21:54:56.262 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.147+0000 7ff51d6fe880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379696148491, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 19, "file_size": 4582858, "table_properties": {"data_size": 4575682, "index_size": 5520, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 837, "raw_key_size": 6762, "raw_average_key_size": 22, "raw_value_size": 4566772, "raw_average_value_size": 15022, "num_data_blocks": 175, "num_entries": 304, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1708379696, "oldest_key_time": 3, "file_creation_time": 0}} 2024-02-19T21:54:56.262 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.147+0000 7ff51d6fe880 4 rocksdb: [version_set.cc:3826] Creating manifest 20 2024-02-19T21:54:56.262 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.263 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.147+0000 7ff51d6fe880 4 rocksdb: [version_set.cc:3291] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-19T21:54:56.263 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.150+0000 7ff51d6fe880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1708379696151183, "job": 1, "event": "recovery_finished"} 2024-02-19T21:54:56.263 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.151+0000 7ff51d6fe880 4 rocksdb: DB pointer 0x56322cdd9800 2024-02-19T21:54:56.263 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.151+0000 7ff5044f0700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-02-19T21:54:56.263 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.151+0000 7ff5044f0700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-02-19T21:54:56.263 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ** DB Stats ** 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.264 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ** Compaction Stats [default] ** 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: L0 1/0 4.37 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 706.8 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: L6 1/0 23.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Sum 2/0 28.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 706.8 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 706.8 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ** Compaction Stats [default] ** 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 706.8 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Flush(GB): cumulative 0.004, interval 0.004 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:54:56.265 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Cumulative compaction: 0.00 GB write, 235.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Interval compaction: 0.00 GB write, 235.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ** Compaction Stats [default] ** 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: L0 1/0 4.37 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 706.8 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: L6 1/0 23.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Sum 2/0 28.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 706.8 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:56.266 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ** Compaction Stats [default] ** 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 706.8 0.01 0.00 1 0.006 0 0 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Uptime(secs): 0.0 total, 0.0 interval 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Flush(GB): cumulative 0.004, interval 0.000 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: AddFile(Total Files): cumulative 0, interval 0 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: AddFile(L0 Files): cumulative 0, interval 0 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: AddFile(Keys): cumulative 0, interval 0 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Cumulative compaction: 0.00 GB write, 235.38 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-19T21:54:56.267 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: ** File Read Latency Histogram By Level [default] ** 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.151+0000 7ff51d6fe880 0 starting mon.smithi196 rank 1 at public addrs [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] at bind addrs [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi196 fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.152+0000 7ff51d6fe880 1 mon.smithi196@-1(???) e2 preinit fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.153+0000 7ff51d6fe880 0 mon.smithi196@-1(???).mds e7 new map 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.153+0000 7ff51d6fe880 0 mon.smithi196@-1(???).mds e7 print_map 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: e7 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: enable_multiple, ever_enabled_multiple: 1,1 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: legacy client fscid: 1 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Filesystem 'foofs' (1) 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: fs_name foofs 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: epoch 4 2024-02-19T21:54:56.268 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: flags 12 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: created 2024-02-19T21:51:14.830583+0000 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: modified 2024-02-19T21:51:19.787557+0000 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: tableserver 0 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: root 0 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: session_timeout 60 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: session_autoclose 300 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: max_file_size 1099511627776 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: required_client_features {} 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: last_failure 0 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: last_failure_osd_epoch 0 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: max_mds 1 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: in 0 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: up {0=14474} 2024-02-19T21:54:56.269 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: failed 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: damaged 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: stopped 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: data_pools [3] 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: metadata_pool 2 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: inline_data disabled 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: balancer 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: standby_count_wanted 1 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: [mds.foofs.smithi131.gemwnz{0:14474} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.131:6834/916443639,v1:172.21.15.131:6835/916443639] compat {c=[1],r=[1],i=[7ff]}] 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: Standby daemons: 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: [mds.foofs.smithi196.hhhkwt{-1:24311} state up:standby seq 3 join_fscid=1 addr [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] compat {c=[1],r=[1],i=[7ff]}] 2024-02-19T21:54:56.270 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.153+0000 7ff51d6fe880 0 mon.smithi196@-1(???).osd e70 crush map has features 3314933000854323200, adjusting msgr requires 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.153+0000 7ff51d6fe880 0 mon.smithi196@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.153+0000 7ff51d6fe880 0 mon.smithi196@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.153+0000 7ff51d6fe880 0 mon.smithi196@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:52.504658+0000 mgr.smithi131.cjubsp (mgr.14720) 30 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:54:52.615477+0000 mgr.smithi131.cjubsp (mgr.14720) 31 : cephadm [INF] Upgrade: Updating mon.smithi196 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:54:52.620230+0000 mon.smithi131 (mon.0) 14 : audit [INF] from='mgr.14720 ' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:54:52.621224+0000 mon.smithi196 (mon.1) 35 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:54:52.621911+0000 mon.smithi196 (mon.1) 36 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:54:52.622563+0000 mon.smithi196 (mon.1) 37 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:54:52.623224+0000 mgr.smithi131.cjubsp (mgr.14720) 32 : cephadm [INF] Deploying daemon mon.smithi196 on smithi196 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T21:54:56.153+0000 7ff51d6fe880 1 mon.smithi196@-1(???).paxosservice(auth 1..43) refresh upgraded, format 0 -> 3 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 bash[57353]: 28e82f762666368ff97272cf8bfebd3f63ebe0a5eafe084ee28ed99d0c043c45 2024-02-19T21:54:56.271 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:56 smithi196 systemd[1]: Started Ceph mon.smithi196 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:54:57.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:54:54.402034+0000 mon.smithi131 (mon.0) 15 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:54:57.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:54:54.402355+0000 mon.smithi131 (mon.0) 16 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:54:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:54.402034+0000 mon.smithi131 (mon.0) 15 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi131"}]: dispatch 2024-02-19T21:54:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:54.402355+0000 mon.smithi131 (mon.0) 16 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mon metadata", "id": "smithi196"}]: dispatch 2024-02-19T21:54:58.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:54.505128+0000 mgr.smithi131.cjubsp (mgr.14720) 33 : cluster [DBG] pgmap v20: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:58.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:56.505693+0000 mgr.smithi131.cjubsp (mgr.14720) 34 : cluster [DBG] pgmap v21: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:58.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:57.624728+0000 mon.smithi196 (mon.1) 1 : cluster [INF] mon.smithi196 calling monitor election 2024-02-19T21:54:58.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:57.626011+0000 mon.smithi131 (mon.0) 17 : cluster [INF] mon.smithi131 calling monitor election 2024-02-19T21:54:58.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:57.639572+0000 mon.smithi131 (mon.0) 18 : cluster [INF] mon.smithi131 is new leader, mons smithi131,smithi196 in quorum (ranks 0,1) 2024-02-19T21:54:58.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:57.644132+0000 mon.smithi131 (mon.0) 19 : cluster [DBG] monmap e2: 2 mons at {smithi131=[v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0],smithi196=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} removed_ranks: {} 2024-02-19T21:54:58.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:57.656346+0000 mon.smithi131 (mon.0) 20 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:54:58.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:57.656386+0000 mon.smithi131 (mon.0) 21 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-02-19T21:54:58.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:57.656805+0000 mon.smithi131 (mon.0) 22 : cluster [DBG] mgrmap e33: smithi131.cjubsp(active, since 38s), standbys: smithi196.voprex 2024-02-19T21:54:58.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:57.656961+0000 mon.smithi131 (mon.0) 23 : cluster [INF] overall HEALTH_OK 2024-02-19T21:54:58.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:54:57.661123+0000 mon.smithi131 (mon.0) 24 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:58.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:54:57.662094+0000 mon.smithi131 (mon.0) 25 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:54.505128+0000 mgr.smithi131.cjubsp (mgr.14720) 33 : cluster [DBG] pgmap v20: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:56.505693+0000 mgr.smithi131.cjubsp (mgr.14720) 34 : cluster [DBG] pgmap v21: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:57.624728+0000 mon.smithi196 (mon.1) 1 : cluster [INF] mon.smithi196 calling monitor election 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:57.626011+0000 mon.smithi131 (mon.0) 17 : cluster [INF] mon.smithi131 calling monitor election 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:57.639572+0000 mon.smithi131 (mon.0) 18 : cluster [INF] mon.smithi131 is new leader, mons smithi131,smithi196 in quorum (ranks 0,1) 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:57.644132+0000 mon.smithi131 (mon.0) 19 : cluster [DBG] monmap e2: 2 mons at {smithi131=[v2:172.21.15.131:3300/0,v1:172.21.15.131:6789/0],smithi196=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} removed_ranks: {} 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:57.656346+0000 mon.smithi131 (mon.0) 20 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:57.656386+0000 mon.smithi131 (mon.0) 21 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-02-19T21:54:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:57.656805+0000 mon.smithi131 (mon.0) 22 : cluster [DBG] mgrmap e33: smithi131.cjubsp(active, since 38s), standbys: smithi196.voprex 2024-02-19T21:54:59.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:57.656961+0000 mon.smithi131 (mon.0) 23 : cluster [INF] overall HEALTH_OK 2024-02-19T21:54:59.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:57.661123+0000 mon.smithi131 (mon.0) 24 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:54:59.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:54:57.662094+0000 mon.smithi131 (mon.0) 25 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:54:59.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:54:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:54:58.506458+0000 mgr.smithi131.cjubsp (mgr.14720) 35 : cluster [DBG] pgmap v22: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:00.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:54:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:54:58.506458+0000 mgr.smithi131.cjubsp (mgr.14720) 35 : cluster [DBG] pgmap v22: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:00.071 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:00.507054+0000 mgr.smithi131.cjubsp (mgr.14720) 36 : cluster [DBG] pgmap v23: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.858292+0000 mon.smithi131 (mon.0) 26 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.859365+0000 mon.smithi131 (mon.0) 27 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.860249+0000 mon.smithi131 (mon.0) 28 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.864734+0000 mon.smithi131 (mon.0) 29 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.870815+0000 mon.smithi131 (mon.0) 30 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.874226+0000 mon.smithi131 (mon.0) 31 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.876470+0000 mon.smithi131 (mon.0) 32 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:02.422 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.877709+0000 mon.smithi131 (mon.0) 33 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.881903+0000 mon.smithi131 (mon.0) 34 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.882678+0000 mon.smithi131 (mon.0) 35 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi131"}]: dispatch 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.886042+0000 mon.smithi131 (mon.0) 36 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi131"}]': finished 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.886671+0000 mon.smithi131 (mon.0) 37 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi196"}]: dispatch 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:01.889805+0000 mon.smithi131 (mon.0) 38 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi196"}]': finished 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:00.507054+0000 mgr.smithi131.cjubsp (mgr.14720) 36 : cluster [DBG] pgmap v23: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.858292+0000 mon.smithi131 (mon.0) 26 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.859365+0000 mon.smithi131 (mon.0) 27 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.860249+0000 mon.smithi131 (mon.0) 28 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.864734+0000 mon.smithi131 (mon.0) 29 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.870815+0000 mon.smithi131 (mon.0) 30 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.874226+0000 mon.smithi131 (mon.0) 31 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.876470+0000 mon.smithi131 (mon.0) 32 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.877709+0000 mon.smithi131 (mon.0) 33 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.881903+0000 mon.smithi131 (mon.0) 34 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.882678+0000 mon.smithi131 (mon.0) 35 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi131"}]: dispatch 2024-02-19T21:55:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.886042+0000 mon.smithi131 (mon.0) 36 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi131"}]': finished 2024-02-19T21:55:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.886671+0000 mon.smithi131 (mon.0) 37 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi196"}]: dispatch 2024-02-19T21:55:02.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:01.889805+0000 mon.smithi131 (mon.0) 38 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi196"}]': finished 2024-02-19T21:55:03.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:01.878298+0000 mgr.smithi131.cjubsp (mgr.14720) 37 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-02-19T21:55:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:01.878298+0000 mgr.smithi131.cjubsp (mgr.14720) 37 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-02-19T21:55:04.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:02.507652+0000 mgr.smithi131.cjubsp (mgr.14720) 38 : cluster [DBG] pgmap v24: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:04.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:03.702887+0000 mon.smithi131 (mon.0) 39 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:04.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:03.703578+0000 mon.smithi131 (mon.0) 40 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi131", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:55:04.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:03.704213+0000 mon.smithi131 (mon.0) 41 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:02.507652+0000 mgr.smithi131.cjubsp (mgr.14720) 38 : cluster [DBG] pgmap v24: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:03.702887+0000 mon.smithi131 (mon.0) 39 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:03.703578+0000 mon.smithi131 (mon.0) 40 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi131", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:55:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:03.704213+0000 mon.smithi131 (mon.0) 41 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:05.111 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:03.698661+0000 mgr.smithi131.cjubsp (mgr.14720) 39 : cephadm [INF] Upgrade: Updating crash.smithi131 (1/2) 2024-02-19T21:55:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:03.704720+0000 mgr.smithi131.cjubsp (mgr.14720) 40 : cephadm [INF] Deploying daemon crash.smithi131 on smithi131 2024-02-19T21:55:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:03.698661+0000 mgr.smithi131.cjubsp (mgr.14720) 39 : cephadm [INF] Upgrade: Updating crash.smithi131 (1/2) 2024-02-19T21:55:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:03.704720+0000 mgr.smithi131.cjubsp (mgr.14720) 40 : cephadm [INF] Deploying daemon crash.smithi131 on smithi131 2024-02-19T21:55:06.493 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:04.508000+0000 mgr.smithi131.cjubsp (mgr.14720) 41 : cluster [DBG] pgmap v25: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:06.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:04.508000+0000 mgr.smithi131.cjubsp (mgr.14720) 41 : cluster [DBG] pgmap v25: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:06.508642+0000 mgr.smithi131.cjubsp (mgr.14720) 42 : cluster [DBG] pgmap v26: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:07.306877+0000 mon.smithi131 (mon.0) 42 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:06.508642+0000 mgr.smithi131.cjubsp (mgr.14720) 42 : cluster [DBG] pgmap v26: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:07.306877+0000 mon.smithi131 (mon.0) 42 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:10.098 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:08.509337+0000 mgr.smithi131.cjubsp (mgr.14720) 43 : cluster [DBG] pgmap v27: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:10.098 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:08.765660+0000 mgr.smithi131.cjubsp (mgr.14720) 44 : cephadm [INF] Upgrade: Updating crash.smithi196 (2/2) 2024-02-19T21:55:10.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:08.770589+0000 mon.smithi131 (mon.0) 43 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:10.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:08.771474+0000 mon.smithi131 (mon.0) 44 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi196", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:55:10.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:08.772276+0000 mon.smithi131 (mon.0) 45 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:10.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:08.772913+0000 mgr.smithi131.cjubsp (mgr.14720) 45 : cephadm [INF] Deploying daemon crash.smithi196 on smithi196 2024-02-19T21:55:10.147 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:08.509337+0000 mgr.smithi131.cjubsp (mgr.14720) 43 : cluster [DBG] pgmap v27: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:08.765660+0000 mgr.smithi131.cjubsp (mgr.14720) 44 : cephadm [INF] Upgrade: Updating crash.smithi196 (2/2) 2024-02-19T21:55:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:08.770589+0000 mon.smithi131 (mon.0) 43 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:08.771474+0000 mon.smithi131 (mon.0) 44 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi196", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-19T21:55:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:08.772276+0000 mon.smithi131 (mon.0) 45 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:10.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:08.772913+0000 mgr.smithi131.cjubsp (mgr.14720) 45 : cephadm [INF] Deploying daemon crash.smithi196 on smithi196 2024-02-19T21:55:12.166 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:10.509847+0000 mgr.smithi131.cjubsp (mgr.14720) 46 : cluster [DBG] pgmap v28: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:10.509847+0000 mgr.smithi131.cjubsp (mgr.14720) 46 : cluster [DBG] pgmap v28: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:13.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:12.247371+0000 mon.smithi131 (mon.0) 46 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:13.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:12.248294+0000 mon.smithi131 (mon.0) 47 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:13.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:12.495207+0000 mon.smithi131 (mon.0) 48 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:13.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:12.510442+0000 mgr.smithi131.cjubsp (mgr.14720) 47 : cluster [DBG] pgmap v29: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:13.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:12.247371+0000 mon.smithi131 (mon.0) 46 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:13.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:12.248294+0000 mon.smithi131 (mon.0) 47 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:13.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:12.495207+0000 mon.smithi131 (mon.0) 48 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:13.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:12.510442+0000 mgr.smithi131.cjubsp (mgr.14720) 47 : cluster [DBG] pgmap v29: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:15.179 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:15.709 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:55:15.847 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:14.510814+0000 mgr.smithi131.cjubsp (mgr.14720) 48 : cluster [DBG] pgmap v30: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:15.865 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:14.510814+0000 mgr.smithi131.cjubsp (mgr.14720) 48 : cluster [DBG] pgmap v30: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:16.299 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (8m) 25s ago 9m 21.2M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 starting - - - - 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 starting - - - - 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (7m) 25s ago 9m 34.4M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (3m) 25s ago 3m 39.0M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (3m) 14s ago 3m 13.1M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (64s) 25s ago 11m 428M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (43s) 14s ago 8m 387M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (33s) 25s ago 11m 21.9M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (20s) 14s ago 8m 23.2M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (97s) 25s ago 97s 51.9M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:55:16.300 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (90s) 14s ago 90s 49.4M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (9m) 25s ago 9m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (8m) 14s ago 8m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (8m) 25s ago 8m 373M 1944M 16.2.5 6933c2a0b7dd f42dfded3a0d 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (7m) 25s ago 7m 286M 1944M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (7m) 25s ago 7m 226M 1944M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (6m) 25s ago 6m 340M 1944M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (6m) 14s ago 6m 356M 2712M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (6m) 14s ago 6m 374M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (5m) 14s ago 5m 311M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (5m) 14s ago 5m 467M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:55:16.301 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (69s) 25s ago 8m 50.5M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:55:16.668 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 4, 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 10 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:55:16.669 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:55:16.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:15.698247+0000 mgr.smithi131.cjubsp (mgr.14720) 49 : audit [DBG] from='client.34112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:16.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:16.002257+0000 mgr.smithi131.cjubsp (mgr.14720) 50 : audit [DBG] from='client.34116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:16.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:16.349221+0000 mon.smithi131 (mon.0) 49 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:16.979 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "5/23 daemons upgraded", 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading crash daemons", 2024-02-19T21:55:16.980 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:55:16.981 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:55:17.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:15.698247+0000 mgr.smithi131.cjubsp (mgr.14720) 49 : audit [DBG] from='client.34112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:16.002257+0000 mgr.smithi131.cjubsp (mgr.14720) 50 : audit [DBG] from='client.34116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:17.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:16.349221+0000 mon.smithi131 (mon.0) 49 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:17.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:16.291485+0000 mgr.smithi131.cjubsp (mgr.14720) 51 : audit [DBG] from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:17.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:16.511293+0000 mgr.smithi131.cjubsp (mgr.14720) 52 : cluster [DBG] pgmap v31: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:17.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:16.669076+0000 mon.smithi131 (mon.0) 50 : audit [DBG] from='client.? 172.21.15.131:0/4199051238' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:17.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:16.979196+0000 mgr.smithi131.cjubsp (mgr.14720) 53 : audit [DBG] from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:16.291485+0000 mgr.smithi131.cjubsp (mgr.14720) 51 : audit [DBG] from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:16.511293+0000 mgr.smithi131.cjubsp (mgr.14720) 52 : cluster [DBG] pgmap v31: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:16.669076+0000 mon.smithi131 (mon.0) 50 : audit [DBG] from='client.? 172.21.15.131:0/4199051238' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:18.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:16.979196+0000 mgr.smithi131.cjubsp (mgr.14720) 53 : audit [DBG] from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:18.511639+0000 mgr.smithi131.cjubsp (mgr.14720) 54 : cluster [DBG] pgmap v32: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.075772+0000 mon.smithi131 (mon.0) 51 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.076568+0000 mon.smithi131 (mon.0) 52 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.077170+0000 mon.smithi131 (mon.0) 53 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.082960+0000 mon.smithi131 (mon.0) 54 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.089961+0000 mon.smithi131 (mon.0) 55 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.092613+0000 mon.smithi131 (mon.0) 56 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.094412+0000 mon.smithi131 (mon.0) 57 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:20.099 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.095435+0000 mon.smithi131 (mon.0) 58 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.096495+0000 mon.smithi131 (mon.0) 59 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:19.096997+0000 mgr.smithi131.cjubsp (mgr.14720) 55 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.102202+0000 mon.smithi131 (mon.0) 60 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.102886+0000 mon.smithi131 (mon.0) 61 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi131"}]: dispatch 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.107961+0000 mon.smithi131 (mon.0) 62 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi131"}]': finished 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.108485+0000 mon.smithi131 (mon.0) 63 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi196"}]: dispatch 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.113593+0000 mon.smithi131 (mon.0) 64 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi196"}]': finished 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.114610+0000 mon.smithi131 (mon.0) 65 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:19.114745+0000 mgr.smithi131.cjubsp (mgr.14720) 56 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-19T21:55:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:19.115478+0000 mgr.smithi131.cjubsp (mgr.14720) 57 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-02-19T21:55:20.225 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:20.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:18.511639+0000 mgr.smithi131.cjubsp (mgr.14720) 54 : cluster [DBG] pgmap v32: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:20.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.075772+0000 mon.smithi131 (mon.0) 51 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.076568+0000 mon.smithi131 (mon.0) 52 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.077170+0000 mon.smithi131 (mon.0) 53 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.082960+0000 mon.smithi131 (mon.0) 54 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.089961+0000 mon.smithi131 (mon.0) 55 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.092613+0000 mon.smithi131 (mon.0) 56 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.094412+0000 mon.smithi131 (mon.0) 57 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.095435+0000 mon.smithi131 (mon.0) 58 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.096495+0000 mon.smithi131 (mon.0) 59 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:19.096997+0000 mgr.smithi131.cjubsp (mgr.14720) 55 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.102202+0000 mon.smithi131 (mon.0) 60 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.102886+0000 mon.smithi131 (mon.0) 61 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi131"}]: dispatch 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.107961+0000 mon.smithi131 (mon.0) 62 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi131"}]': finished 2024-02-19T21:55:20.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.108485+0000 mon.smithi131 (mon.0) 63 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi196"}]: dispatch 2024-02-19T21:55:20.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.113593+0000 mon.smithi131 (mon.0) 64 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi196"}]': finished 2024-02-19T21:55:20.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.114610+0000 mon.smithi131 (mon.0) 65 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-19T21:55:20.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:19.114745+0000 mgr.smithi131.cjubsp (mgr.14720) 56 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-19T21:55:20.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:19.115478+0000 mgr.smithi131.cjubsp (mgr.14720) 57 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-02-19T21:55:21.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:20.512152+0000 mgr.smithi131.cjubsp (mgr.14720) 58 : cluster [DBG] pgmap v33: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:21.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:20.565375+0000 mgr.smithi131.cjubsp (mgr.14720) 59 : cephadm [INF] Upgrade: Updating osd.0 2024-02-19T21:55:21.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:20.569641+0000 mon.smithi131 (mon.0) 66 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:21.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:20.570292+0000 mon.smithi131 (mon.0) 67 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-19T21:55:21.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:20.570773+0000 mon.smithi131 (mon.0) 68 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:21.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:20.571795+0000 mgr.smithi131.cjubsp (mgr.14720) 60 : cephadm [INF] Deploying daemon osd.0 on smithi131 2024-02-19T21:55:21.971 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:20.512152+0000 mgr.smithi131.cjubsp (mgr.14720) 58 : cluster [DBG] pgmap v33: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:21.971 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:20.565375+0000 mgr.smithi131.cjubsp (mgr.14720) 59 : cephadm [INF] Upgrade: Updating osd.0 2024-02-19T21:55:21.971 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:20.569641+0000 mon.smithi131 (mon.0) 66 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:21.971 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:20.570292+0000 mon.smithi131 (mon.0) 67 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-19T21:55:21.972 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:20.570773+0000 mon.smithi131 (mon.0) 68 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:21.972 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:20.571795+0000 mgr.smithi131.cjubsp (mgr.14720) 60 : cephadm [INF] Deploying daemon osd.0 on smithi131 2024-02-19T21:55:23.691 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:23 smithi131 systemd[1]: Stopping Ceph osd.0 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:55:23.691 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:22.512774+0000 mgr.smithi131.cjubsp (mgr.14720) 61 : cluster [DBG] pgmap v34: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:23.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:22.512774+0000 mgr.smithi131.cjubsp (mgr.14720) 61 : cluster [DBG] pgmap v34: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:55:24.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.0[44669]: debug 2024-02-19T21:55:23.690+0000 7f1e1a5f3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T21:55:24.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.0[44669]: debug 2024-02-19T21:55:23.690+0000 7f1e1a5f3700 -1 osd.0 70 *** Got signal Terminated *** 2024-02-19T21:55:24.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.0[44669]: debug 2024-02-19T21:55:23.690+0000 7f1e1a5f3700 -1 osd.0 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T21:55:24.505 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:24 smithi131 bash[79723]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.0 2024-02-19T21:55:24.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.866328+0000 mon.smithi131 (mon.0) 69 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.866356+0000 mon.smithi131 (mon.0) 70 : cluster [INF] osd.0 failed (root=default,host=smithi131) (connection refused reported by osd.6) 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.892457+0000 mon.smithi131 (mon.0) 71 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969051+0000 mon.smithi131 (mon.0) 72 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969148+0000 mon.smithi131 (mon.0) 73 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969233+0000 mon.smithi131 (mon.0) 74 : cluster [DBG] osd.0 reported immediately failed by osd.2 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969321+0000 mon.smithi131 (mon.0) 75 : cluster [DBG] osd.0 reported immediately failed by osd.1 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969411+0000 mon.smithi131 (mon.0) 76 : cluster [DBG] osd.0 reported immediately failed by osd.2 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969491+0000 mon.smithi131 (mon.0) 77 : cluster [DBG] osd.0 reported immediately failed by osd.1 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969562+0000 mon.smithi131 (mon.0) 78 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:24.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969633+0000 mon.smithi131 (mon.0) 79 : cluster [DBG] osd.0 reported immediately failed by osd.5 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969712+0000 mon.smithi131 (mon.0) 80 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969780+0000 mon.smithi131 (mon.0) 81 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969863+0000 mon.smithi131 (mon.0) 82 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969930+0000 mon.smithi131 (mon.0) 83 : cluster [DBG] osd.0 reported immediately failed by osd.5 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.969999+0000 mon.smithi131 (mon.0) 84 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.970065+0000 mon.smithi131 (mon.0) 85 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:23.970137+0000 mon.smithi131 (mon.0) 86 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.067627+0000 mon.smithi131 (mon.0) 87 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.093042+0000 mon.smithi131 (mon.0) 88 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.168651+0000 mon.smithi131 (mon.0) 89 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.369517+0000 mon.smithi131 (mon.0) 90 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370074+0000 mon.smithi131 (mon.0) 91 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370187+0000 mon.smithi131 (mon.0) 92 : cluster [DBG] osd.0 reported immediately failed by osd.2 2024-02-19T21:55:24.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370231+0000 mon.smithi131 (mon.0) 93 : cluster [DBG] osd.0 reported immediately failed by osd.1 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370329+0000 mon.smithi131 (mon.0) 94 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370404+0000 mon.smithi131 (mon.0) 95 : cluster [DBG] osd.0 reported immediately failed by osd.2 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370451+0000 mon.smithi131 (mon.0) 96 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370546+0000 mon.smithi131 (mon.0) 97 : cluster [DBG] osd.0 reported immediately failed by osd.1 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370622+0000 mon.smithi131 (mon.0) 98 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370692+0000 mon.smithi131 (mon.0) 99 : cluster [DBG] osd.0 reported immediately failed by osd.5 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370760+0000 mon.smithi131 (mon.0) 100 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370840+0000 mon.smithi131 (mon.0) 101 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370917+0000 mon.smithi131 (mon.0) 102 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:24.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.370985+0000 mon.smithi131 (mon.0) 103 : cluster [DBG] osd.0 reported immediately failed by osd.5 2024-02-19T21:55:24.927 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.371051+0000 mon.smithi131 (mon.0) 104 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:24.927 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.371117+0000 mon.smithi131 (mon.0) 105 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:24.927 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.468488+0000 mon.smithi131 (mon.0) 106 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:24.927 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.494111+0000 mon.smithi131 (mon.0) 107 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:25.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.866328+0000 mon.smithi131 (mon.0) 69 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.866356+0000 mon.smithi131 (mon.0) 70 : cluster [INF] osd.0 failed (root=default,host=smithi131) (connection refused reported by osd.6) 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.892457+0000 mon.smithi131 (mon.0) 71 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969051+0000 mon.smithi131 (mon.0) 72 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969148+0000 mon.smithi131 (mon.0) 73 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969233+0000 mon.smithi131 (mon.0) 74 : cluster [DBG] osd.0 reported immediately failed by osd.2 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969321+0000 mon.smithi131 (mon.0) 75 : cluster [DBG] osd.0 reported immediately failed by osd.1 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969411+0000 mon.smithi131 (mon.0) 76 : cluster [DBG] osd.0 reported immediately failed by osd.2 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969491+0000 mon.smithi131 (mon.0) 77 : cluster [DBG] osd.0 reported immediately failed by osd.1 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969562+0000 mon.smithi131 (mon.0) 78 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969633+0000 mon.smithi131 (mon.0) 79 : cluster [DBG] osd.0 reported immediately failed by osd.5 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969712+0000 mon.smithi131 (mon.0) 80 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969780+0000 mon.smithi131 (mon.0) 81 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969863+0000 mon.smithi131 (mon.0) 82 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969930+0000 mon.smithi131 (mon.0) 83 : cluster [DBG] osd.0 reported immediately failed by osd.5 2024-02-19T21:55:25.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.969999+0000 mon.smithi131 (mon.0) 84 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.970065+0000 mon.smithi131 (mon.0) 85 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:23.970137+0000 mon.smithi131 (mon.0) 86 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.067627+0000 mon.smithi131 (mon.0) 87 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.093042+0000 mon.smithi131 (mon.0) 88 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.168651+0000 mon.smithi131 (mon.0) 89 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.369517+0000 mon.smithi131 (mon.0) 90 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370074+0000 mon.smithi131 (mon.0) 91 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370187+0000 mon.smithi131 (mon.0) 92 : cluster [DBG] osd.0 reported immediately failed by osd.2 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370231+0000 mon.smithi131 (mon.0) 93 : cluster [DBG] osd.0 reported immediately failed by osd.1 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370329+0000 mon.smithi131 (mon.0) 94 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370404+0000 mon.smithi131 (mon.0) 95 : cluster [DBG] osd.0 reported immediately failed by osd.2 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370451+0000 mon.smithi131 (mon.0) 96 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:25.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370546+0000 mon.smithi131 (mon.0) 97 : cluster [DBG] osd.0 reported immediately failed by osd.1 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370622+0000 mon.smithi131 (mon.0) 98 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370692+0000 mon.smithi131 (mon.0) 99 : cluster [DBG] osd.0 reported immediately failed by osd.5 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370760+0000 mon.smithi131 (mon.0) 100 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370840+0000 mon.smithi131 (mon.0) 101 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370917+0000 mon.smithi131 (mon.0) 102 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.370985+0000 mon.smithi131 (mon.0) 103 : cluster [DBG] osd.0 reported immediately failed by osd.5 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.371051+0000 mon.smithi131 (mon.0) 104 : cluster [DBG] osd.0 reported immediately failed by osd.7 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.371117+0000 mon.smithi131 (mon.0) 105 : cluster [DBG] osd.0 reported immediately failed by osd.4 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.468488+0000 mon.smithi131 (mon.0) 106 : cluster [DBG] osd.0 reported immediately failed by osd.6 2024-02-19T21:55:25.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.494111+0000 mon.smithi131 (mon.0) 107 : cluster [DBG] osd.0 reported immediately failed by osd.3 2024-02-19T21:55:25.269 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:25.834 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.513180+0000 mgr.smithi131.cjubsp (mgr.14720) 62 : cluster [DBG] pgmap v35: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:25.834 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.576211+0000 mon.smithi131 (mon.0) 108 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:55:25.834 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:24.584380+0000 mon.smithi131 (mon.0) 109 : cluster [DBG] osdmap e71: 8 total, 7 up, 8 in 2024-02-19T21:55:25.834 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:24.584660+0000 mon.smithi131 (mon.0) 110 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-19T21:55:25.834 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:24.585244+0000 mon.smithi131 (mon.0) 111 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-02-19T21:55:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.513180+0000 mgr.smithi131.cjubsp (mgr.14720) 62 : cluster [DBG] pgmap v35: 129 pgs: 129 active+clean; 303 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.576211+0000 mon.smithi131 (mon.0) 108 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:55:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:24.584380+0000 mon.smithi131 (mon.0) 109 : cluster [DBG] osdmap e71: 8 total, 7 up, 8 in 2024-02-19T21:55:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:24.584660+0000 mon.smithi131 (mon.0) 110 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-19T21:55:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:24.585244+0000 mon.smithi131 (mon.0) 111 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-02-19T21:55:26.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:26 smithi131 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.0.service: Succeeded. 2024-02-19T21:55:26.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:26 smithi131 systemd[1]: Stopped Ceph osd.0 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:55:26.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:26 smithi131 systemd[1]: Starting Ceph osd.0 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:55:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:25.584578+0000 mon.smithi131 (mon.0) 112 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-02-19T21:55:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:25.584708+0000 mon.smithi131 (mon.0) 113 : cluster [DBG] osdmap e72: 8 total, 7 up, 8 in 2024-02-19T21:55:27.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:25.584578+0000 mon.smithi131 (mon.0) 112 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-02-19T21:55:27.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:25.584708+0000 mon.smithi131 (mon.0) 113 : cluster [DBG] osdmap e72: 8 total, 7 up, 8 in 2024-02-19T21:55:27.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:27 smithi131 bash[80046]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-02-19T21:55:27.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:27 smithi131 bash[80046]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-19T21:55:27.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:27 smithi131 bash[80046]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-02-19T21:55:27.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:27 smithi131 bash[80046]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-02-19T21:55:27.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:27 smithi131 bash[80046]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-02-19T21:55:27.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:27 smithi131 bash[80046]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-19T21:55:27.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:27 smithi131 bash[80046]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-19T21:55:27.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:27 smithi131 bash[80046]: --> ceph-volume lvm activate successful for osd ID: 0 2024-02-19T21:55:27.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:26.514074+0000 mgr.smithi131.cjubsp (mgr.14720) 63 : cluster [DBG] pgmap v38: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:27.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:26.583041+0000 mon.smithi131 (mon.0) 114 : cluster [WRN] Health check failed: Degraded data redundancy: 182/1353 objects degraded (13.452%), 29 pgs degraded (PG_DEGRADED) 2024-02-19T21:55:28.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:26.514074+0000 mgr.smithi131.cjubsp (mgr.14720) 63 : cluster [DBG] pgmap v38: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:28.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:26.583041+0000 mon.smithi131 (mon.0) 114 : cluster [WRN] Health check failed: Degraded data redundancy: 182/1353 objects degraded (13.452%), 29 pgs degraded (PG_DEGRADED) 2024-02-19T21:55:28.756 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-0[80308]: debug 2024-02-19T21:55:28.442+0000 7f67d5fcd700 1 -- 172.21.15.131:0/1940786575 <== mon.0 v2:172.21.15.131:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5604501b6680 con 0x56045017d000 2024-02-19T21:55:28.756 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:28 smithi131 bash[80293]: a0e5546e07ca2daf62796c247f64706d0ae1d2473b3bdcb8803c7421f4db9454 2024-02-19T21:55:28.756 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:28 smithi131 systemd[1]: Started Ceph osd.0 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:55:29.256 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-0[80308]: debug 2024-02-19T21:55:28.966+0000 7f67dda5b380 -1 Falling back to public interface 2024-02-19T21:55:29.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:28.514741+0000 mgr.smithi131.cjubsp (mgr.14720) 64 : cluster [DBG] pgmap v39: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:29.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:28.582519+0000 mon.smithi131 (mon.0) 115 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:29.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:28.583297+0000 mon.smithi131 (mon.0) 116 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:28.514741+0000 mgr.smithi131.cjubsp (mgr.14720) 64 : cluster [DBG] pgmap v39: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:28.582519+0000 mon.smithi131 (mon.0) 115 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:30.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:28.583297+0000 mon.smithi131 (mon.0) 116 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:30.316 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:31.892 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:30.515191+0000 mgr.smithi131.cjubsp (mgr.14720) 65 : cluster [DBG] pgmap v40: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:31.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:30.515191+0000 mgr.smithi131.cjubsp (mgr.14720) 65 : cluster [DBG] pgmap v40: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:33.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:32.516140+0000 mgr.smithi131.cjubsp (mgr.14720) 66 : cluster [DBG] pgmap v41: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:34.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:32.516140+0000 mgr.smithi131.cjubsp (mgr.14720) 66 : cluster [DBG] pgmap v41: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:34.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-0[80308]: debug 2024-02-19T21:55:33.653+0000 7f67dda5b380 -1 osd.0 70 log_to_monitors {default=true} 2024-02-19T21:55:34.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:33.657329+0000 mon.smithi131 (mon.0) 117 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/1629117177,v1:172.21.15.131:6803/1629117177]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-19T21:55:35.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 21:55:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-0[80308]: debug 2024-02-19T21:55:34.610+0000 7f67cd73f700 -1 osd.0 70 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-19T21:55:35.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:33.657329+0000 mon.smithi131 (mon.0) 117 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/1629117177,v1:172.21.15.131:6803/1629117177]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-19T21:55:35.354 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:35.842 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:34.516589+0000 mgr.smithi131.cjubsp (mgr.14720) 67 : cluster [DBG] pgmap v42: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 206 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:35.842 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:34.594023+0000 mon.smithi131 (mon.0) 118 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/1629117177,v1:172.21.15.131:6803/1629117177]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-19T21:55:35.842 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:34.594066+0000 mon.smithi131 (mon.0) 119 : cluster [DBG] osdmap e73: 8 total, 7 up, 8 in 2024-02-19T21:55:35.842 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:34.600272+0000 mon.smithi131 (mon.0) 120 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/1629117177,v1:172.21.15.131:6803/1629117177]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:55:35.842 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.518593+0000 mon.smithi131 (mon.0) 121 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:35.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:34.516589+0000 mgr.smithi131.cjubsp (mgr.14720) 67 : cluster [DBG] pgmap v42: 129 pgs: 29 active+undersized+degraded, 20 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 206 B/s wr, 0 op/s; 182/1353 objects degraded (13.452%) 2024-02-19T21:55:35.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:34.594023+0000 mon.smithi131 (mon.0) 118 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/1629117177,v1:172.21.15.131:6803/1629117177]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-19T21:55:35.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:34.594066+0000 mon.smithi131 (mon.0) 119 : cluster [DBG] osdmap e73: 8 total, 7 up, 8 in 2024-02-19T21:55:35.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:34.600272+0000 mon.smithi131 (mon.0) 120 : audit [INF] from='osd.0 [v2:172.21.15.131:6802/1629117177,v1:172.21.15.131:6803/1629117177]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:55:35.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.518593+0000 mon.smithi131 (mon.0) 121 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:35.592540+0000 mon.smithi131 (mon.0) 122 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:55:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:35.603263+0000 mon.smithi131 (mon.0) 123 : cluster [INF] osd.0 [v2:172.21.15.131:6802/1629117177,v1:172.21.15.131:6803/1629117177] boot 2024-02-19T21:55:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:35.603331+0000 mon.smithi131 (mon.0) 124 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-02-19T21:55:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.603724+0000 mon.smithi131 (mon.0) 125 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:55:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.674604+0000 mon.smithi131 (mon.0) 126 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:36.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.819934+0000 mon.smithi131 (mon.0) 127 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.820572+0000 mon.smithi131 (mon.0) 128 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.826865+0000 mon.smithi131 (mon.0) 129 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.834212+0000 mon.smithi131 (mon.0) 130 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.837400+0000 mon.smithi131 (mon.0) 131 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.839524+0000 mon.smithi131 (mon.0) 132 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.840773+0000 mon.smithi131 (mon.0) 133 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.841942+0000 mon.smithi131 (mon.0) 134 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.843104+0000 mon.smithi131 (mon.0) 135 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:35.843234+0000 mgr.smithi131.cjubsp (mgr.14720) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-19T21:55:36.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:35.843833+0000 mgr.smithi131.cjubsp (mgr.14720) 69 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-02-19T21:55:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:35.592540+0000 mon.smithi131 (mon.0) 122 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:55:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:35.603263+0000 mon.smithi131 (mon.0) 123 : cluster [INF] osd.0 [v2:172.21.15.131:6802/1629117177,v1:172.21.15.131:6803/1629117177] boot 2024-02-19T21:55:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:35.603331+0000 mon.smithi131 (mon.0) 124 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-02-19T21:55:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.603724+0000 mon.smithi131 (mon.0) 125 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-19T21:55:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.674604+0000 mon.smithi131 (mon.0) 126 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.819934+0000 mon.smithi131 (mon.0) 127 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:37.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.820572+0000 mon.smithi131 (mon.0) 128 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.826865+0000 mon.smithi131 (mon.0) 129 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.834212+0000 mon.smithi131 (mon.0) 130 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.837400+0000 mon.smithi131 (mon.0) 131 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.839524+0000 mon.smithi131 (mon.0) 132 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.840773+0000 mon.smithi131 (mon.0) 133 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.841942+0000 mon.smithi131 (mon.0) 134 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.843104+0000 mon.smithi131 (mon.0) 135 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:35.843234+0000 mgr.smithi131.cjubsp (mgr.14720) 68 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-19T21:55:37.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:35.843833+0000 mgr.smithi131.cjubsp (mgr.14720) 69 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-02-19T21:55:37.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:36.517214+0000 mgr.smithi131.cjubsp (mgr.14720) 70 : cluster [DBG] pgmap v45: 129 pgs: 33 peering, 9 active+undersized+degraded, 7 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 72/1353 objects degraded (5.322%) 2024-02-19T21:55:37.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:36.606437+0000 mon.smithi131 (mon.0) 136 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-02-19T21:55:37.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:36.833025+0000 mon.smithi131 (mon.0) 137 : cluster [WRN] Health check update: Degraded data redundancy: 72/1353 objects degraded (5.322%), 9 pgs degraded (PG_DEGRADED) 2024-02-19T21:55:38.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:36.517214+0000 mgr.smithi131.cjubsp (mgr.14720) 70 : cluster [DBG] pgmap v45: 129 pgs: 33 peering, 9 active+undersized+degraded, 7 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 72/1353 objects degraded (5.322%) 2024-02-19T21:55:38.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:36.606437+0000 mon.smithi131 (mon.0) 136 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-02-19T21:55:38.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:36.833025+0000 mon.smithi131 (mon.0) 137 : cluster [WRN] Health check update: Degraded data redundancy: 72/1353 objects degraded (5.322%), 9 pgs degraded (PG_DEGRADED) 2024-02-19T21:55:39.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:38.517684+0000 mgr.smithi131.cjubsp (mgr.14720) 71 : cluster [DBG] pgmap v47: 129 pgs: 33 peering, 9 active+undersized+degraded, 7 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 72/1353 objects degraded (5.322%) 2024-02-19T21:55:40.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:38.517684+0000 mgr.smithi131.cjubsp (mgr.14720) 71 : cluster [DBG] pgmap v47: 129 pgs: 33 peering, 9 active+undersized+degraded, 7 active+undersized, 80 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 72/1353 objects degraded (5.322%) 2024-02-19T21:55:40.387 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:40.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:40.605983+0000 mon.smithi131 (mon.0) 138 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 72/1353 objects degraded (5.322%), 9 pgs degraded) 2024-02-19T21:55:40.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:40.606035+0000 mon.smithi131 (mon.0) 139 : cluster [INF] Cluster is now healthy 2024-02-19T21:55:41.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:40.605983+0000 mon.smithi131 (mon.0) 138 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 72/1353 objects degraded (5.322%), 9 pgs degraded) 2024-02-19T21:55:41.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:40.606035+0000 mon.smithi131 (mon.0) 139 : cluster [INF] Cluster is now healthy 2024-02-19T21:55:41.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:40.518305+0000 mgr.smithi131.cjubsp (mgr.14720) 72 : cluster [DBG] pgmap v48: 129 pgs: 33 peering, 96 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:42.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:40.518305+0000 mgr.smithi131.cjubsp (mgr.14720) 72 : cluster [DBG] pgmap v48: 129 pgs: 33 peering, 96 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:43.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:42.518935+0000 mgr.smithi131.cjubsp (mgr.14720) 73 : cluster [DBG] pgmap v49: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 258 B/s rd, 258 B/s wr, 0 op/s 2024-02-19T21:55:44.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:42.518935+0000 mgr.smithi131.cjubsp (mgr.14720) 73 : cluster [DBG] pgmap v49: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 258 B/s rd, 258 B/s wr, 0 op/s 2024-02-19T21:55:45.435 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:45.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:44.519422+0000 mgr.smithi131.cjubsp (mgr.14720) 74 : cluster [DBG] pgmap v50: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 229 B/s rd, 229 B/s wr, 0 op/s 2024-02-19T21:55:46.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:44.519422+0000 mgr.smithi131.cjubsp (mgr.14720) 74 : cluster [DBG] pgmap v50: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 229 B/s rd, 229 B/s wr, 0 op/s 2024-02-19T21:55:47.298 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:55:47.869 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (8m) 12s ago 10m 21.1M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (41s) 12s ago 10m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (36s) 31s ago 9m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (8m) 12s ago 9m 34.4M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (4m) 12s ago 4m 39.1M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (4m) 31s ago 4m 13.2M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (96s) 12s ago 11m 434M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (75s) 31s ago 9m 387M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (65s) 12s ago 11m 39.7M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (52s) 31s ago 9m 26.9M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (2m) 12s ago 2m 77.8M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (2m) 31s ago 2m 49.5M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (9m) 12s ago 9m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:55:47.870 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (8m) 31s ago 8m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (19s) 12s ago 8m 13.1M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (8m) 12s ago 8m 286M 1944M 16.2.5 6933c2a0b7dd 1388bde78aa9 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (7m) 12s ago 7m 227M 1944M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (7m) 12s ago 7m 341M 1944M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (7m) 31s ago 7m 356M 2712M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (6m) 31s ago 6m 375M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (6m) 31s ago 6m 311M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (6m) 31s ago 6m 467M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:55:47.871 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (100s) 12s ago 9m 51.1M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:55:47.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:46.520350+0000 mgr.smithi131.cjubsp (mgr.14720) 75 : cluster [DBG] pgmap v51: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:55:48.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:46.520350+0000 mgr.smithi131.cjubsp (mgr.14720) 75 : cluster [DBG] pgmap v51: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:55:48.216 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 1, 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 7 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 5, 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 9 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:55:48.217 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:55:48.507 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:55:48.507 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:55:48.507 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:55:48.507 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:55:48.507 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:55:48.507 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T21:55:48.507 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:55:48.507 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:55:48.508 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:55:48.508 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "7/23 daemons upgraded", 2024-02-19T21:55:48.508 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading osd daemons", 2024-02-19T21:55:48.508 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:55:48.508 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:55:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:47.285713+0000 mgr.smithi131.cjubsp (mgr.14720) 76 : audit [DBG] from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:47.577315+0000 mgr.smithi131.cjubsp (mgr.14720) 77 : audit [DBG] from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:47.861486+0000 mgr.smithi131.cjubsp (mgr.14720) 78 : audit [DBG] from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:49.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:48.217191+0000 mon.smithi131 (mon.0) 140 : audit [DBG] from='client.? 172.21.15.131:0/3211899219' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:49.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:47.285713+0000 mgr.smithi131.cjubsp (mgr.14720) 76 : audit [DBG] from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:49.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:47.577315+0000 mgr.smithi131.cjubsp (mgr.14720) 77 : audit [DBG] from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:49.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:47.861486+0000 mgr.smithi131.cjubsp (mgr.14720) 78 : audit [DBG] from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:49.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:48.217191+0000 mon.smithi131 (mon.0) 140 : audit [DBG] from='client.? 172.21.15.131:0/3211899219' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:55:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:48.507615+0000 mgr.smithi131.cjubsp (mgr.14720) 79 : audit [DBG] from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:48.520691+0000 mgr.smithi131.cjubsp (mgr.14720) 80 : cluster [DBG] pgmap v52: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 171 B/s wr, 0 op/s 2024-02-19T21:55:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:48.507615+0000 mgr.smithi131.cjubsp (mgr.14720) 79 : audit [DBG] from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:55:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:48.520691+0000 mgr.smithi131.cjubsp (mgr.14720) 80 : cluster [DBG] pgmap v52: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 171 B/s wr, 0 op/s 2024-02-19T21:55:50.482 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:50.521206+0000 mgr.smithi131.cjubsp (mgr.14720) 81 : cluster [DBG] pgmap v53: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:50.675910+0000 mon.smithi131 (mon.0) 141 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:50.716059+0000 mon.smithi131 (mon.0) 142 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:50.859282+0000 mon.smithi131 (mon.0) 143 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-19T21:55:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:50.859504+0000 mgr.smithi131.cjubsp (mgr.14720) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-19T21:55:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:50.860725+0000 mgr.smithi131.cjubsp (mgr.14720) 83 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-02-19T21:55:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:50.521206+0000 mgr.smithi131.cjubsp (mgr.14720) 81 : cluster [DBG] pgmap v53: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:55:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:50.675910+0000 mon.smithi131 (mon.0) 141 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:50.716059+0000 mon.smithi131 (mon.0) 142 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:50.859282+0000 mon.smithi131 (mon.0) 143 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-19T21:55:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:50.859504+0000 mgr.smithi131.cjubsp (mgr.14720) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-19T21:55:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:50.860725+0000 mgr.smithi131.cjubsp (mgr.14720) 83 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-02-19T21:55:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:52.433701+0000 mgr.smithi131.cjubsp (mgr.14720) 84 : cephadm [INF] Upgrade: Updating osd.1 2024-02-19T21:55:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:52.438223+0000 mon.smithi131 (mon.0) 144 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:52.438914+0000 mon.smithi131 (mon.0) 145 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-19T21:55:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:55:52.439439+0000 mon.smithi131 (mon.0) 146 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:55:52.440493+0000 mgr.smithi131.cjubsp (mgr.14720) 85 : cephadm [INF] Deploying daemon osd.1 on smithi131 2024-02-19T21:55:53.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:52.521784+0000 mgr.smithi131.cjubsp (mgr.14720) 86 : cluster [DBG] pgmap v54: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-02-19T21:55:53.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:52.683944+0000 mon.smithi131 (mon.0) 147 : cluster [DBG] mgrmap e34: smithi131.cjubsp(active, since 93s), standbys: smithi196.voprex 2024-02-19T21:55:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:52.433701+0000 mgr.smithi131.cjubsp (mgr.14720) 84 : cephadm [INF] Upgrade: Updating osd.1 2024-02-19T21:55:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:52.438223+0000 mon.smithi131 (mon.0) 144 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:55:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:52.438914+0000 mon.smithi131 (mon.0) 145 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-19T21:55:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:55:52.439439+0000 mon.smithi131 (mon.0) 146 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:55:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:55:52.440493+0000 mgr.smithi131.cjubsp (mgr.14720) 85 : cephadm [INF] Deploying daemon osd.1 on smithi131 2024-02-19T21:55:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:52.521784+0000 mgr.smithi131.cjubsp (mgr.14720) 86 : cluster [DBG] pgmap v54: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-02-19T21:55:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:52.683944+0000 mon.smithi131 (mon.0) 147 : cluster [DBG] mgrmap e34: smithi131.cjubsp(active, since 93s), standbys: smithi196.voprex 2024-02-19T21:55:55.527 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:55:55.575 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:55 smithi131 systemd[1]: Stopping Ceph osd.1 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:55:55.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:54.522160+0000 mgr.smithi131.cjubsp (mgr.14720) 87 : cluster [DBG] pgmap v55: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T21:55:55.943 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:54.522160+0000 mgr.smithi131.cjubsp (mgr.14720) 87 : cluster [DBG] pgmap v55: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T21:55:55.943 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1[47703]: debug 2024-02-19T21:55:55.595+0000 7fb9d1ddd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T21:55:55.943 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1[47703]: debug 2024-02-19T21:55:55.595+0000 7fb9d1ddd700 -1 osd.1 75 *** Got signal Terminated *** 2024-02-19T21:55:55.943 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1[47703]: debug 2024-02-19T21:55:55.595+0000 7fb9d1ddd700 -1 osd.1 75 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T21:55:56.255 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:56 smithi131 bash[82213]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1 2024-02-19T21:55:56.547 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:56 smithi131 bash[82285]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1 2024-02-19T21:55:56.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.859216+0000 mon.smithi131 (mon.0) 148 : cluster [DBG] osd.1 reported immediately failed by osd.0 2024-02-19T21:55:56.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.859265+0000 mon.smithi131 (mon.0) 149 : cluster [INF] osd.1 failed (root=default,host=smithi131) (connection refused reported by osd.0) 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.859500+0000 mon.smithi131 (mon.0) 150 : cluster [DBG] osd.1 reported immediately failed by osd.2 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.859578+0000 mon.smithi131 (mon.0) 151 : cluster [DBG] osd.1 reported immediately failed by osd.4 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.859754+0000 mon.smithi131 (mon.0) 152 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.859888+0000 mon.smithi131 (mon.0) 153 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.860017+0000 mon.smithi131 (mon.0) 154 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.860136+0000 mon.smithi131 (mon.0) 155 : cluster [DBG] osd.1 reported immediately failed by osd.4 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.860260+0000 mon.smithi131 (mon.0) 156 : cluster [DBG] osd.1 reported immediately failed by osd.0 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.860387+0000 mon.smithi131 (mon.0) 157 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.860502+0000 mon.smithi131 (mon.0) 158 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.860618+0000 mon.smithi131 (mon.0) 159 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.860749+0000 mon.smithi131 (mon.0) 160 : cluster [DBG] osd.1 reported immediately failed by osd.2 2024-02-19T21:55:56.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.860884+0000 mon.smithi131 (mon.0) 161 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:55.861004+0000 mon.smithi131 (mon.0) 162 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260121+0000 mon.smithi131 (mon.0) 163 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260269+0000 mon.smithi131 (mon.0) 164 : cluster [DBG] osd.1 reported immediately failed by osd.0 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260342+0000 mon.smithi131 (mon.0) 165 : cluster [DBG] osd.1 reported immediately failed by osd.2 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260410+0000 mon.smithi131 (mon.0) 166 : cluster [DBG] osd.1 reported immediately failed by osd.4 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260472+0000 mon.smithi131 (mon.0) 167 : cluster [DBG] osd.1 reported immediately failed by osd.4 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260606+0000 mon.smithi131 (mon.0) 168 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260676+0000 mon.smithi131 (mon.0) 169 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260738+0000 mon.smithi131 (mon.0) 170 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.260810+0000 mon.smithi131 (mon.0) 171 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.261409+0000 mon.smithi131 (mon.0) 172 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.261480+0000 mon.smithi131 (mon.0) 173 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.261633+0000 mon.smithi131 (mon.0) 174 : cluster [DBG] osd.1 reported immediately failed by osd.2 2024-02-19T21:55:56.925 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.261742+0000 mon.smithi131 (mon.0) 175 : cluster [DBG] osd.1 reported immediately failed by osd.0 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.261887+0000 mon.smithi131 (mon.0) 176 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.352172+0000 mon.smithi131 (mon.0) 177 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.352330+0000 mon.smithi131 (mon.0) 178 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.352455+0000 mon.smithi131 (mon.0) 179 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.352882+0000 mon.smithi131 (mon.0) 180 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.552636+0000 mon.smithi131 (mon.0) 181 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.553041+0000 mon.smithi131 (mon.0) 182 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.553320+0000 mon.smithi131 (mon.0) 183 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:56.926 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.553441+0000 mon.smithi131 (mon.0) 184 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.859216+0000 mon.smithi131 (mon.0) 148 : cluster [DBG] osd.1 reported immediately failed by osd.0 2024-02-19T21:55:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.859265+0000 mon.smithi131 (mon.0) 149 : cluster [INF] osd.1 failed (root=default,host=smithi131) (connection refused reported by osd.0) 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.859500+0000 mon.smithi131 (mon.0) 150 : cluster [DBG] osd.1 reported immediately failed by osd.2 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.859578+0000 mon.smithi131 (mon.0) 151 : cluster [DBG] osd.1 reported immediately failed by osd.4 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.859754+0000 mon.smithi131 (mon.0) 152 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.859888+0000 mon.smithi131 (mon.0) 153 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.860017+0000 mon.smithi131 (mon.0) 154 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.860136+0000 mon.smithi131 (mon.0) 155 : cluster [DBG] osd.1 reported immediately failed by osd.4 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.860260+0000 mon.smithi131 (mon.0) 156 : cluster [DBG] osd.1 reported immediately failed by osd.0 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.860387+0000 mon.smithi131 (mon.0) 157 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.860502+0000 mon.smithi131 (mon.0) 158 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.860618+0000 mon.smithi131 (mon.0) 159 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.860749+0000 mon.smithi131 (mon.0) 160 : cluster [DBG] osd.1 reported immediately failed by osd.2 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.860884+0000 mon.smithi131 (mon.0) 161 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:57.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:55.861004+0000 mon.smithi131 (mon.0) 162 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260121+0000 mon.smithi131 (mon.0) 163 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260269+0000 mon.smithi131 (mon.0) 164 : cluster [DBG] osd.1 reported immediately failed by osd.0 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260342+0000 mon.smithi131 (mon.0) 165 : cluster [DBG] osd.1 reported immediately failed by osd.2 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260410+0000 mon.smithi131 (mon.0) 166 : cluster [DBG] osd.1 reported immediately failed by osd.4 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260472+0000 mon.smithi131 (mon.0) 167 : cluster [DBG] osd.1 reported immediately failed by osd.4 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260606+0000 mon.smithi131 (mon.0) 168 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260676+0000 mon.smithi131 (mon.0) 169 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260738+0000 mon.smithi131 (mon.0) 170 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.260810+0000 mon.smithi131 (mon.0) 171 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.261409+0000 mon.smithi131 (mon.0) 172 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.261480+0000 mon.smithi131 (mon.0) 173 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.261633+0000 mon.smithi131 (mon.0) 174 : cluster [DBG] osd.1 reported immediately failed by osd.2 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.261742+0000 mon.smithi131 (mon.0) 175 : cluster [DBG] osd.1 reported immediately failed by osd.0 2024-02-19T21:55:57.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.261887+0000 mon.smithi131 (mon.0) 176 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:57.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.352172+0000 mon.smithi131 (mon.0) 177 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:57.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.352330+0000 mon.smithi131 (mon.0) 178 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:57.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.352455+0000 mon.smithi131 (mon.0) 179 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:57.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.352882+0000 mon.smithi131 (mon.0) 180 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:57.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.552636+0000 mon.smithi131 (mon.0) 181 : cluster [DBG] osd.1 reported immediately failed by osd.5 2024-02-19T21:55:57.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.553041+0000 mon.smithi131 (mon.0) 182 : cluster [DBG] osd.1 reported immediately failed by osd.3 2024-02-19T21:55:57.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.553320+0000 mon.smithi131 (mon.0) 183 : cluster [DBG] osd.1 reported immediately failed by osd.6 2024-02-19T21:55:57.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.553441+0000 mon.smithi131 (mon.0) 184 : cluster [DBG] osd.1 reported immediately failed by osd.7 2024-02-19T21:55:57.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.522947+0000 mgr.smithi131.cjubsp (mgr.14720) 88 : cluster [DBG] pgmap v56: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 1 op/s 2024-02-19T21:55:57.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.576697+0000 mon.smithi131 (mon.0) 185 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:55:57.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:56.585220+0000 mon.smithi131 (mon.0) 186 : cluster [DBG] osdmap e76: 8 total, 7 up, 8 in 2024-02-19T21:55:57.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.522947+0000 mgr.smithi131.cjubsp (mgr.14720) 88 : cluster [DBG] pgmap v56: 129 pgs: 129 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 1 op/s 2024-02-19T21:55:57.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.576697+0000 mon.smithi131 (mon.0) 185 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:55:57.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:56.585220+0000 mon.smithi131 (mon.0) 186 : cluster [DBG] osdmap e76: 8 total, 7 up, 8 in 2024-02-19T21:55:58.507 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:58 smithi131 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.1.service: Succeeded. 2024-02-19T21:55:58.507 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:58 smithi131 systemd[1]: Stopped Ceph osd.1 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:55:58.507 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:58 smithi131 systemd[1]: Starting Ceph osd.1 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:55:58.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:57.584866+0000 mon.smithi131 (mon.0) 187 : cluster [DBG] osdmap e77: 8 total, 7 up, 8 in 2024-02-19T21:55:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:57.584866+0000 mon.smithi131 (mon.0) 187 : cluster [DBG] osdmap e77: 8 total, 7 up, 8 in 2024-02-19T21:55:59.483 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:59 smithi131 bash[82547]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-02-19T21:55:59.483 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:59 smithi131 bash[82547]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-19T21:55:59.483 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:59 smithi131 bash[82547]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-02-19T21:55:59.484 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:59 smithi131 bash[82547]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-02-19T21:55:59.484 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:59 smithi131 bash[82547]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-02-19T21:55:59.484 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:59 smithi131 bash[82547]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-19T21:55:59.484 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:59 smithi131 bash[82547]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-19T21:55:59.484 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:55:59 smithi131 bash[82547]: --> ceph-volume lvm activate successful for osd ID: 1 2024-02-19T21:55:59.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:55:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:55:58.523386+0000 mgr.smithi131.cjubsp (mgr.14720) 89 : cluster [DBG] pgmap v59: 129 pgs: 17 stale+active+clean, 112 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 639 B/s wr, 1 op/s 2024-02-19T21:55:59.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:55:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:55:58.523386+0000 mgr.smithi131.cjubsp (mgr.14720) 89 : cluster [DBG] pgmap v59: 129 pgs: 17 stale+active+clean, 112 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 639 B/s wr, 1 op/s 2024-02-19T21:56:00.563 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:00.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:00.598738+0000 mon.smithi131 (mon.0) 188 : cluster [WRN] Health check failed: Degraded data redundancy: 16/1353 objects degraded (1.183%), 3 pgs degraded (PG_DEGRADED) 2024-02-19T21:56:00.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:00.625902+0000 mon.smithi131 (mon.0) 189 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:00.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:00.626689+0000 mon.smithi131 (mon.0) 190 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:56:00.756 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:56:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-1[82812]: debug 2024-02-19T21:56:00.453+0000 7f9da1650700 1 -- 172.21.15.131:0/3548630129 <== mon.0 v2:172.21.15.131:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ae087fa680 con 0x55ae087c1400 2024-02-19T21:56:00.756 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:56:00 smithi131 bash[82797]: f8444cfee8d5e5004bba439d473bb06c679e055b7a8061faf20ae685f60b51df 2024-02-19T21:56:00.757 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:56:00 smithi131 systemd[1]: Started Ceph osd.1 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:56:00.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:00.598738+0000 mon.smithi131 (mon.0) 188 : cluster [WRN] Health check failed: Degraded data redundancy: 16/1353 objects degraded (1.183%), 3 pgs degraded (PG_DEGRADED) 2024-02-19T21:56:00.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:00.625902+0000 mon.smithi131 (mon.0) 189 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:00.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:00.626689+0000 mon.smithi131 (mon.0) 190 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:56:01.256 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:56:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-1[82812]: debug 2024-02-19T21:56:00.980+0000 7f9da90de380 -1 Falling back to public interface 2024-02-19T21:56:01.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:00.523831+0000 mgr.smithi131.cjubsp (mgr.14720) 90 : cluster [DBG] pgmap v60: 129 pgs: 3 active+undersized+degraded, 16 stale+active+clean, 2 active+undersized, 108 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 16/1353 objects degraded (1.183%) 2024-02-19T21:56:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:00.523831+0000 mgr.smithi131.cjubsp (mgr.14720) 90 : cluster [DBG] pgmap v60: 129 pgs: 3 active+undersized+degraded, 16 stale+active+clean, 2 active+undersized, 108 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 16/1353 objects degraded (1.183%) 2024-02-19T21:56:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:02.524726+0000 mgr.smithi131.cjubsp (mgr.14720) 91 : cluster [DBG] pgmap v61: 129 pgs: 27 active+undersized+degraded, 26 active+undersized, 76 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 150/1353 objects degraded (11.086%) 2024-02-19T21:56:04.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:02.524726+0000 mgr.smithi131.cjubsp (mgr.14720) 91 : cluster [DBG] pgmap v61: 129 pgs: 27 active+undersized+degraded, 26 active+undersized, 76 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 150/1353 objects degraded (11.086%) 2024-02-19T21:56:05.506 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:56:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-1[82812]: debug 2024-02-19T21:56:05.229+0000 7f9da90de380 -1 osd.1 75 log_to_monitors {default=true} 2024-02-19T21:56:05.597 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:06.057 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 21:56:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-1[82812]: debug 2024-02-19T21:56:05.820+0000 7f9d9bdc8700 -1 osd.1 75 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-19T21:56:06.058 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:04.525176+0000 mgr.smithi131.cjubsp (mgr.14720) 92 : cluster [DBG] pgmap v62: 129 pgs: 27 active+undersized+degraded, 26 active+undersized, 76 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 150/1353 objects degraded (11.086%) 2024-02-19T21:56:06.058 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:05.232588+0000 mon.smithi131 (mon.0) 191 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/1995452000,v1:172.21.15.131:6811/1995452000]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-19T21:56:06.058 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:05.675327+0000 mon.smithi131 (mon.0) 192 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:04.525176+0000 mgr.smithi131.cjubsp (mgr.14720) 92 : cluster [DBG] pgmap v62: 129 pgs: 27 active+undersized+degraded, 26 active+undersized, 76 active+clean; 303 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 150/1353 objects degraded (11.086%) 2024-02-19T21:56:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:05.232588+0000 mon.smithi131 (mon.0) 191 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/1995452000,v1:172.21.15.131:6811/1995452000]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-19T21:56:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:05.675327+0000 mon.smithi131 (mon.0) 192 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:05.813216+0000 mon.smithi131 (mon.0) 193 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/1995452000,v1:172.21.15.131:6811/1995452000]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-19T21:56:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:05.813260+0000 mon.smithi131 (mon.0) 194 : cluster [DBG] osdmap e78: 8 total, 7 up, 8 in 2024-02-19T21:56:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:05.813728+0000 mon.smithi131 (mon.0) 195 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/1995452000,v1:172.21.15.131:6811/1995452000]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:56:07.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:05.813216+0000 mon.smithi131 (mon.0) 193 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/1995452000,v1:172.21.15.131:6811/1995452000]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-19T21:56:07.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:05.813260+0000 mon.smithi131 (mon.0) 194 : cluster [DBG] osdmap e78: 8 total, 7 up, 8 in 2024-02-19T21:56:07.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:05.813728+0000 mon.smithi131 (mon.0) 195 : audit [INF] from='osd.1 [v2:172.21.15.131:6810/1995452000,v1:172.21.15.131:6811/1995452000]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:56:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:06.525801+0000 mgr.smithi131.cjubsp (mgr.14720) 93 : cluster [DBG] pgmap v64: 129 pgs: 27 active+undersized+degraded, 26 active+undersized, 76 active+clean; 303 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 229 B/s wr, 0 op/s; 150/1353 objects degraded (11.086%) 2024-02-19T21:56:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:06.811523+0000 mon.smithi131 (mon.0) 196 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:56:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:06.815491+0000 mon.smithi131 (mon.0) 197 : cluster [INF] osd.1 [v2:172.21.15.131:6810/1995452000,v1:172.21.15.131:6811/1995452000] boot 2024-02-19T21:56:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:06.815549+0000 mon.smithi131 (mon.0) 198 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-02-19T21:56:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:06.816498+0000 mon.smithi131 (mon.0) 199 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.321810+0000 mon.smithi131 (mon.0) 200 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.322659+0000 mon.smithi131 (mon.0) 201 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.323237+0000 mon.smithi131 (mon.0) 202 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.327030+0000 mon.smithi131 (mon.0) 203 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.331910+0000 mon.smithi131 (mon.0) 204 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.334542+0000 mon.smithi131 (mon.0) 205 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.336360+0000 mon.smithi131 (mon.0) 206 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.337389+0000 mon.smithi131 (mon.0) 207 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.338376+0000 mon.smithi131 (mon.0) 208 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.339411+0000 mon.smithi131 (mon.0) 209 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-19T21:56:08.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:07.801212+0000 mon.smithi131 (mon.0) 210 : cluster [WRN] Health check update: Degraded data redundancy: 150/1353 objects degraded (11.086%), 27 pgs degraded (PG_DEGRADED) 2024-02-19T21:56:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:06.525801+0000 mgr.smithi131.cjubsp (mgr.14720) 93 : cluster [DBG] pgmap v64: 129 pgs: 27 active+undersized+degraded, 26 active+undersized, 76 active+clean; 303 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 229 B/s wr, 0 op/s; 150/1353 objects degraded (11.086%) 2024-02-19T21:56:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:06.811523+0000 mon.smithi131 (mon.0) 196 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:56:08.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:06.815491+0000 mon.smithi131 (mon.0) 197 : cluster [INF] osd.1 [v2:172.21.15.131:6810/1995452000,v1:172.21.15.131:6811/1995452000] boot 2024-02-19T21:56:08.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:06.815549+0000 mon.smithi131 (mon.0) 198 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-02-19T21:56:08.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:06.816498+0000 mon.smithi131 (mon.0) 199 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-19T21:56:08.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.321810+0000 mon.smithi131 (mon.0) 200 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:08.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.322659+0000 mon.smithi131 (mon.0) 201 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:56:08.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.323237+0000 mon.smithi131 (mon.0) 202 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:56:08.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.327030+0000 mon.smithi131 (mon.0) 203 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:08.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.331910+0000 mon.smithi131 (mon.0) 204 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:08.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.334542+0000 mon.smithi131 (mon.0) 205 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:56:08.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.336360+0000 mon.smithi131 (mon.0) 206 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:08.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.337389+0000 mon.smithi131 (mon.0) 207 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:08.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.338376+0000 mon.smithi131 (mon.0) 208 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:08.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.339411+0000 mon.smithi131 (mon.0) 209 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-19T21:56:08.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:07.801212+0000 mon.smithi131 (mon.0) 210 : cluster [WRN] Health check update: Degraded data redundancy: 150/1353 objects degraded (11.086%), 27 pgs degraded (PG_DEGRADED) 2024-02-19T21:56:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:07.339537+0000 mgr.smithi131.cjubsp (mgr.14720) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-19T21:56:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:56:07.340132+0000 mgr.smithi131.cjubsp (mgr.14720) 95 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-02-19T21:56:09.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:07.831100+0000 mon.smithi131 (mon.0) 211 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-02-19T21:56:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:07.339537+0000 mgr.smithi131.cjubsp (mgr.14720) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-19T21:56:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:56:07.340132+0000 mgr.smithi131.cjubsp (mgr.14720) 95 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-02-19T21:56:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:07.831100+0000 mon.smithi131 (mon.0) 211 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-02-19T21:56:10.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:08.526248+0000 mgr.smithi131.cjubsp (mgr.14720) 96 : cluster [DBG] pgmap v67: 129 pgs: 27 active+undersized+degraded, 26 active+undersized, 76 active+clean; 303 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 150/1353 objects degraded (11.086%) 2024-02-19T21:56:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:08.526248+0000 mgr.smithi131.cjubsp (mgr.14720) 96 : cluster [DBG] pgmap v67: 129 pgs: 27 active+undersized+degraded, 26 active+undersized, 76 active+clean; 303 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 150/1353 objects degraded (11.086%) 2024-02-19T21:56:10.649 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:12.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:10.526924+0000 mgr.smithi131.cjubsp (mgr.14720) 97 : cluster [DBG] pgmap v68: 129 pgs: 25 active+undersized+degraded, 24 active+undersized, 80 active+clean; 303 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 137/1353 objects degraded (10.126%) 2024-02-19T21:56:12.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:10.730915+0000 mon.smithi131 (mon.0) 212 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:10.526924+0000 mgr.smithi131.cjubsp (mgr.14720) 97 : cluster [DBG] pgmap v68: 129 pgs: 25 active+undersized+degraded, 24 active+undersized, 80 active+clean; 303 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 137/1353 objects degraded (10.126%) 2024-02-19T21:56:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:10.730915+0000 mon.smithi131 (mon.0) 212 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:13.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:12.730673+0000 mon.smithi131 (mon.0) 213 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 137/1353 objects degraded (10.126%), 25 pgs degraded) 2024-02-19T21:56:13.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:12.730702+0000 mon.smithi131 (mon.0) 214 : cluster [INF] Cluster is now healthy 2024-02-19T21:56:13.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:12.730673+0000 mon.smithi131 (mon.0) 213 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 137/1353 objects degraded (10.126%), 25 pgs degraded) 2024-02-19T21:56:13.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:12.730702+0000 mon.smithi131 (mon.0) 214 : cluster [INF] Cluster is now healthy 2024-02-19T21:56:14.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:12.527832+0000 mgr.smithi131.cjubsp (mgr.14720) 98 : cluster [DBG] pgmap v69: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 304 B/s rd, 304 B/s wr, 0 op/s 2024-02-19T21:56:14.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:12.527832+0000 mgr.smithi131.cjubsp (mgr.14720) 98 : cluster [DBG] pgmap v69: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 304 B/s rd, 304 B/s wr, 0 op/s 2024-02-19T21:56:15.697 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:16.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:14.528189+0000 mgr.smithi131.cjubsp (mgr.14720) 99 : cluster [DBG] pgmap v70: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2024-02-19T21:56:16.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:14.528189+0000 mgr.smithi131.cjubsp (mgr.14720) 99 : cluster [DBG] pgmap v70: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2024-02-19T21:56:18.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:16.528957+0000 mgr.smithi131.cjubsp (mgr.14720) 100 : cluster [DBG] pgmap v71: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 210 B/s rd, 210 B/s wr, 0 op/s 2024-02-19T21:56:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:16.528957+0000 mgr.smithi131.cjubsp (mgr.14720) 100 : cluster [DBG] pgmap v71: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 210 B/s rd, 210 B/s wr, 0 op/s 2024-02-19T21:56:18.826 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:56:19.401 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:56:19.401 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (9m) 12s ago 10m 21.1M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:56:19.401 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (72s) 12s ago 10m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T21:56:19.401 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (67s) 63s ago 9m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T21:56:19.401 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (9m) 12s ago 10m 34.4M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:56:19.401 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (5m) 12s ago 5m 39.0M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (4m) 63s ago 4m 13.2M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (2m) 12s ago 12m 436M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (106s) 63s ago 9m 387M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (97s) 12s ago 12m 41.8M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (83s) 63s ago 9m 26.9M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (2m) 12s ago 2m 78.0M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (2m) 63s ago 2m 49.5M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (10m) 12s ago 10m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (9m) 63s ago 9m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (51s) 12s ago 9m 112M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (19s) 12s ago 8m 13.3M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (8m) 12s ago 8m 227M 1944M 16.2.5 6933c2a0b7dd c3f3f6164204 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (7m) 12s ago 7m 341M 1944M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:56:19.402 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (7m) 63s ago 7m 356M 2712M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:56:19.403 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (7m) 63s ago 7m 375M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:56:19.403 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (7m) 63s ago 6m 311M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:56:19.403 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (6m) 63s ago 6m 467M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:56:19.403 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (2m) 12s ago 10m 57.7M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2, 2024-02-19T21:56:19.742 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 6 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 6, 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 8 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:56:19.743 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:56:20.035 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:56:20.035 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:56:20.035 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:56:20.035 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:56:20.035 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:56:20.035 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T21:56:20.035 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:56:20.035 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:56:20.036 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:56:20.036 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "8/23 daemons upgraded", 2024-02-19T21:56:20.036 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading osd daemons", 2024-02-19T21:56:20.036 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:56:20.036 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:56:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:18.529430+0000 mgr.smithi131.cjubsp (mgr.14720) 101 : cluster [DBG] pgmap v72: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 191 B/s rd, 191 B/s wr, 0 op/s 2024-02-19T21:56:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:18.815092+0000 mgr.smithi131.cjubsp (mgr.14720) 102 : audit [DBG] from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:19.107859+0000 mgr.smithi131.cjubsp (mgr.14720) 103 : audit [DBG] from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:20.100 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:19.743016+0000 mon.smithi131 (mon.0) 215 : audit [DBG] from='client.? 172.21.15.131:0/3260177746' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:18.529430+0000 mgr.smithi131.cjubsp (mgr.14720) 101 : cluster [DBG] pgmap v72: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 191 B/s rd, 191 B/s wr, 0 op/s 2024-02-19T21:56:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:18.815092+0000 mgr.smithi131.cjubsp (mgr.14720) 102 : audit [DBG] from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:19.107859+0000 mgr.smithi131.cjubsp (mgr.14720) 103 : audit [DBG] from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:19.743016+0000 mon.smithi131 (mon.0) 215 : audit [DBG] from='client.? 172.21.15.131:0/3260177746' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:20.732 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:19.393103+0000 mgr.smithi131.cjubsp (mgr.14720) 104 : audit [DBG] from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:20.035903+0000 mgr.smithi131.cjubsp (mgr.14720) 105 : audit [DBG] from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:20.676089+0000 mon.smithi131 (mon.0) 216 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:21.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:20.741742+0000 mon.smithi131 (mon.0) 217 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:19.393103+0000 mgr.smithi131.cjubsp (mgr.14720) 104 : audit [DBG] from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:20.035903+0000 mgr.smithi131.cjubsp (mgr.14720) 105 : audit [DBG] from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:20.676089+0000 mon.smithi131 (mon.0) 216 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:21.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:20.741742+0000 mon.smithi131 (mon.0) 217 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:20.530022+0000 mgr.smithi131.cjubsp (mgr.14720) 106 : cluster [DBG] pgmap v73: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:56:22.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:20.530022+0000 mgr.smithi131.cjubsp (mgr.14720) 106 : cluster [DBG] pgmap v73: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:56:23.145 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:22.352226+0000 mon.smithi131 (mon.0) 218 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-19T21:56:23.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:22.352226+0000 mon.smithi131 (mon.0) 218 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-19T21:56:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:22.352444+0000 mgr.smithi131.cjubsp (mgr.14720) 107 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-19T21:56:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:56:22.353639+0000 mgr.smithi131.cjubsp (mgr.14720) 108 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-02-19T21:56:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:22.530542+0000 mgr.smithi131.cjubsp (mgr.14720) 109 : cluster [DBG] pgmap v74: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:56:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:22.352444+0000 mgr.smithi131.cjubsp (mgr.14720) 107 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-19T21:56:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:56:22.353639+0000 mgr.smithi131.cjubsp (mgr.14720) 108 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-02-19T21:56:24.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:22.530542+0000 mgr.smithi131.cjubsp (mgr.14720) 109 : cluster [DBG] pgmap v74: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:56:25.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:56:23.884701+0000 mgr.smithi131.cjubsp (mgr.14720) 110 : cephadm [INF] Upgrade: Updating osd.2 2024-02-19T21:56:25.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:23.891105+0000 mon.smithi131 (mon.0) 219 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:25.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:23.891737+0000 mon.smithi131 (mon.0) 220 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-19T21:56:25.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:23.892254+0000 mon.smithi131 (mon.0) 221 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:56:25.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:56:23.893261+0000 mgr.smithi131.cjubsp (mgr.14720) 111 : cephadm [INF] Deploying daemon osd.2 on smithi131 2024-02-19T21:56:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:56:23.884701+0000 mgr.smithi131.cjubsp (mgr.14720) 110 : cephadm [INF] Upgrade: Updating osd.2 2024-02-19T21:56:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:23.891105+0000 mon.smithi131 (mon.0) 219 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:23.891737+0000 mon.smithi131 (mon.0) 220 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-19T21:56:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:23.892254+0000 mon.smithi131 (mon.0) 221 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:56:25.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:56:23.893261+0000 mgr.smithi131.cjubsp (mgr.14720) 111 : cephadm [INF] Deploying daemon osd.2 on smithi131 2024-02-19T21:56:25.779 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:24.531071+0000 mgr.smithi131.cjubsp (mgr.14720) 112 : cluster [DBG] pgmap v75: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:56:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:24.531071+0000 mgr.smithi131.cjubsp (mgr.14720) 112 : cluster [DBG] pgmap v75: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:56:27.000 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:26 smithi131 systemd[1]: Stopping Ceph osd.2 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:56:27.256 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2[51279]: debug 2024-02-19T21:56:26.998+0000 7efc5fe8b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T21:56:27.256 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2[51279]: debug 2024-02-19T21:56:26.998+0000 7efc5fe8b700 -1 osd.2 80 *** Got signal Terminated *** 2024-02-19T21:56:27.256 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2[51279]: debug 2024-02-19T21:56:26.998+0000 7efc5fe8b700 -1 osd.2 80 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T21:56:27.756 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:27 smithi131 bash[84662]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2 2024-02-19T21:56:27.756 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:27 smithi131 bash[84718]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2 2024-02-19T21:56:28.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:26.531622+0000 mgr.smithi131.cjubsp (mgr.14720) 113 : cluster [DBG] pgmap v76: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.255381+0000 mon.smithi131 (mon.0) 222 : cluster [DBG] osd.2 reported immediately failed by osd.1 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.255418+0000 mon.smithi131 (mon.0) 223 : cluster [INF] osd.2 failed (root=default,host=smithi131) (connection refused reported by osd.1) 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.255628+0000 mon.smithi131 (mon.0) 224 : cluster [DBG] osd.2 reported immediately failed by osd.0 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.255750+0000 mon.smithi131 (mon.0) 225 : cluster [DBG] osd.2 reported immediately failed by osd.1 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.255892+0000 mon.smithi131 (mon.0) 226 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.256011+0000 mon.smithi131 (mon.0) 227 : cluster [DBG] osd.2 reported immediately failed by osd.0 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.256132+0000 mon.smithi131 (mon.0) 228 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.256379+0000 mon.smithi131 (mon.0) 229 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.256507+0000 mon.smithi131 (mon.0) 230 : cluster [DBG] osd.2 reported immediately failed by osd.7 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.256630+0000 mon.smithi131 (mon.0) 231 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.256765+0000 mon.smithi131 (mon.0) 232 : cluster [DBG] osd.2 reported immediately failed by osd.6 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.256911+0000 mon.smithi131 (mon.0) 233 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.257033+0000 mon.smithi131 (mon.0) 234 : cluster [DBG] osd.2 reported immediately failed by osd.7 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.257189+0000 mon.smithi131 (mon.0) 235 : cluster [DBG] osd.2 reported immediately failed by osd.6 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.257312+0000 mon.smithi131 (mon.0) 236 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.368022+0000 mon.smithi131 (mon.0) 237 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.656150+0000 mon.smithi131 (mon.0) 238 : cluster [DBG] osd.2 reported immediately failed by osd.1 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.656300+0000 mon.smithi131 (mon.0) 239 : cluster [DBG] osd.2 reported immediately failed by osd.0 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.656425+0000 mon.smithi131 (mon.0) 240 : cluster [DBG] osd.2 reported immediately failed by osd.1 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.656542+0000 mon.smithi131 (mon.0) 241 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.656658+0000 mon.smithi131 (mon.0) 242 : cluster [DBG] osd.2 reported immediately failed by osd.0 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.656775+0000 mon.smithi131 (mon.0) 243 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.656903+0000 mon.smithi131 (mon.0) 244 : cluster [DBG] osd.2 reported immediately failed by osd.7 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.657028+0000 mon.smithi131 (mon.0) 245 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.657145+0000 mon.smithi131 (mon.0) 246 : cluster [DBG] osd.2 reported immediately failed by osd.6 2024-02-19T21:56:28.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.657266+0000 mon.smithi131 (mon.0) 247 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.657382+0000 mon.smithi131 (mon.0) 248 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.657513+0000 mon.smithi131 (mon.0) 249 : cluster [DBG] osd.2 reported immediately failed by osd.6 2024-02-19T21:56:28.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.657635+0000 mon.smithi131 (mon.0) 250 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.657743+0000 mon.smithi131 (mon.0) 251 : cluster [DBG] osd.2 reported immediately failed by osd.7 2024-02-19T21:56:28.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.868730+0000 mon.smithi131 (mon.0) 252 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.894304+0000 mon.smithi131 (mon.0) 253 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:26.531622+0000 mgr.smithi131.cjubsp (mgr.14720) 113 : cluster [DBG] pgmap v76: 129 pgs: 129 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.255381+0000 mon.smithi131 (mon.0) 222 : cluster [DBG] osd.2 reported immediately failed by osd.1 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.255418+0000 mon.smithi131 (mon.0) 223 : cluster [INF] osd.2 failed (root=default,host=smithi131) (connection refused reported by osd.1) 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.255628+0000 mon.smithi131 (mon.0) 224 : cluster [DBG] osd.2 reported immediately failed by osd.0 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.255750+0000 mon.smithi131 (mon.0) 225 : cluster [DBG] osd.2 reported immediately failed by osd.1 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.255892+0000 mon.smithi131 (mon.0) 226 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.256011+0000 mon.smithi131 (mon.0) 227 : cluster [DBG] osd.2 reported immediately failed by osd.0 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.256132+0000 mon.smithi131 (mon.0) 228 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.256379+0000 mon.smithi131 (mon.0) 229 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.256507+0000 mon.smithi131 (mon.0) 230 : cluster [DBG] osd.2 reported immediately failed by osd.7 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.256630+0000 mon.smithi131 (mon.0) 231 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.256765+0000 mon.smithi131 (mon.0) 232 : cluster [DBG] osd.2 reported immediately failed by osd.6 2024-02-19T21:56:28.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.256911+0000 mon.smithi131 (mon.0) 233 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.257033+0000 mon.smithi131 (mon.0) 234 : cluster [DBG] osd.2 reported immediately failed by osd.7 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.257189+0000 mon.smithi131 (mon.0) 235 : cluster [DBG] osd.2 reported immediately failed by osd.6 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.257312+0000 mon.smithi131 (mon.0) 236 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.368022+0000 mon.smithi131 (mon.0) 237 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.656150+0000 mon.smithi131 (mon.0) 238 : cluster [DBG] osd.2 reported immediately failed by osd.1 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.656300+0000 mon.smithi131 (mon.0) 239 : cluster [DBG] osd.2 reported immediately failed by osd.0 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.656425+0000 mon.smithi131 (mon.0) 240 : cluster [DBG] osd.2 reported immediately failed by osd.1 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.656542+0000 mon.smithi131 (mon.0) 241 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.656658+0000 mon.smithi131 (mon.0) 242 : cluster [DBG] osd.2 reported immediately failed by osd.0 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.656775+0000 mon.smithi131 (mon.0) 243 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.656903+0000 mon.smithi131 (mon.0) 244 : cluster [DBG] osd.2 reported immediately failed by osd.7 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.657028+0000 mon.smithi131 (mon.0) 245 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.657145+0000 mon.smithi131 (mon.0) 246 : cluster [DBG] osd.2 reported immediately failed by osd.6 2024-02-19T21:56:28.258 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.657266+0000 mon.smithi131 (mon.0) 247 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.657382+0000 mon.smithi131 (mon.0) 248 : cluster [DBG] osd.2 reported immediately failed by osd.4 2024-02-19T21:56:28.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.657513+0000 mon.smithi131 (mon.0) 249 : cluster [DBG] osd.2 reported immediately failed by osd.6 2024-02-19T21:56:28.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.657635+0000 mon.smithi131 (mon.0) 250 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.657743+0000 mon.smithi131 (mon.0) 251 : cluster [DBG] osd.2 reported immediately failed by osd.7 2024-02-19T21:56:28.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.868730+0000 mon.smithi131 (mon.0) 252 : cluster [DBG] osd.2 reported immediately failed by osd.5 2024-02-19T21:56:28.259 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.894304+0000 mon.smithi131 (mon.0) 253 : cluster [DBG] osd.2 reported immediately failed by osd.3 2024-02-19T21:56:29.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.897938+0000 mon.smithi131 (mon.0) 254 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:56:29.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:27.910076+0000 mon.smithi131 (mon.0) 255 : cluster [DBG] osdmap e81: 8 total, 7 up, 8 in 2024-02-19T21:56:29.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.897938+0000 mon.smithi131 (mon.0) 254 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:56:29.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:27.910076+0000 mon.smithi131 (mon.0) 255 : cluster [DBG] osdmap e81: 8 total, 7 up, 8 in 2024-02-19T21:56:29.910 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:29 smithi131 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.2.service: Succeeded. 2024-02-19T21:56:29.910 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:29 smithi131 systemd[1]: Stopped Ceph osd.2 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:56:29.910 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:29 smithi131 systemd[1]: Starting Ceph osd.2 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:56:30.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:28.532005+0000 mgr.smithi131.cjubsp (mgr.14720) 114 : cluster [DBG] pgmap v78: 129 pgs: 10 stale+active+clean, 119 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:56:30.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:28.910214+0000 mon.smithi131 (mon.0) 256 : cluster [DBG] osdmap e82: 8 total, 7 up, 8 in 2024-02-19T21:56:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:28.532005+0000 mgr.smithi131.cjubsp (mgr.14720) 114 : cluster [DBG] pgmap v78: 129 pgs: 10 stale+active+clean, 119 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:56:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:28.910214+0000 mon.smithi131 (mon.0) 256 : cluster [DBG] osdmap e82: 8 total, 7 up, 8 in 2024-02-19T21:56:30.821 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:30.964 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:30 smithi131 bash[84980]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-02-19T21:56:30.964 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:30 smithi131 bash[84980]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-19T21:56:30.964 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:30 smithi131 bash[84980]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-02-19T21:56:30.964 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:30 smithi131 bash[84980]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-02-19T21:56:30.964 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:30 smithi131 bash[84980]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-02-19T21:56:30.964 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:30 smithi131 bash[84980]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-19T21:56:30.964 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:30 smithi131 bash[84980]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-19T21:56:30.964 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:30 smithi131 bash[84980]: --> ceph-volume lvm activate successful for osd ID: 2 2024-02-19T21:56:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:30.910243+0000 mon.smithi131 (mon.0) 257 : cluster [WRN] Health check failed: Degraded data redundancy: 15/1353 objects degraded (1.109%), 2 pgs degraded (PG_DEGRADED) 2024-02-19T21:56:31.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:30.910243+0000 mon.smithi131 (mon.0) 257 : cluster [WRN] Health check failed: Degraded data redundancy: 15/1353 objects degraded (1.109%), 2 pgs degraded (PG_DEGRADED) 2024-02-19T21:56:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:30.532428+0000 mgr.smithi131.cjubsp (mgr.14720) 115 : cluster [DBG] pgmap v80: 129 pgs: 2 active+undersized+degraded, 8 stale+active+clean, 4 active+undersized, 115 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 15/1353 objects degraded (1.109%) 2024-02-19T21:56:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:31.876164+0000 mon.smithi131 (mon.0) 258 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:32.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:31.876874+0000 mon.smithi131 (mon.0) 259 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:56:32.006 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-2[85278]: debug 2024-02-19T21:56:31.750+0000 7f38d6fb1700 1 -- 172.21.15.131:0/4092129150 <== mon.0 v2:172.21.15.131:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56064c70c680 con 0x56064c6d3400 2024-02-19T21:56:32.006 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:31 smithi131 bash[85263]: 433a290cc652f229cb2cd74c129bebfb877fbb25998b7476745e8ba7d8d916ee 2024-02-19T21:56:32.006 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:31 smithi131 systemd[1]: Started Ceph osd.2 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:56:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:30.532428+0000 mgr.smithi131.cjubsp (mgr.14720) 115 : cluster [DBG] pgmap v80: 129 pgs: 2 active+undersized+degraded, 8 stale+active+clean, 4 active+undersized, 115 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 15/1353 objects degraded (1.109%) 2024-02-19T21:56:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:31.876164+0000 mon.smithi131 (mon.0) 258 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:31.876874+0000 mon.smithi131 (mon.0) 259 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:56:32.756 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-2[85278]: debug 2024-02-19T21:56:32.266+0000 7f38dea3f380 -1 Falling back to public interface 2024-02-19T21:56:33.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:32.122029+0000 mon.smithi131 (mon.0) 260 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:33.443 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:32.122029+0000 mon.smithi131 (mon.0) 260 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:34.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:32.533113+0000 mgr.smithi131.cjubsp (mgr.14720) 116 : cluster [DBG] pgmap v81: 129 pgs: 20 active+undersized+degraded, 15 active+undersized, 94 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 125/1353 objects degraded (9.239%) 2024-02-19T21:56:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:32.533113+0000 mgr.smithi131.cjubsp (mgr.14720) 116 : cluster [DBG] pgmap v81: 129 pgs: 20 active+undersized+degraded, 15 active+undersized, 94 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 125/1353 objects degraded (9.239%) 2024-02-19T21:56:35.871 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:36.127 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-2[85278]: debug 2024-02-19T21:56:35.797+0000 7f38dea3f380 -1 osd.2 80 log_to_monitors {default=true} 2024-02-19T21:56:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:34.533833+0000 mgr.smithi131.cjubsp (mgr.14720) 117 : cluster [DBG] pgmap v82: 129 pgs: 20 active+undersized+degraded, 15 active+undersized, 94 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 125/1353 objects degraded (9.239%) 2024-02-19T21:56:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:35.677040+0000 mon.smithi131 (mon.0) 261 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:35.801370+0000 mon.smithi131 (mon.0) 262 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/1836168183,v1:172.21.15.131:6819/1836168183]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-19T21:56:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:34.533833+0000 mgr.smithi131.cjubsp (mgr.14720) 117 : cluster [DBG] pgmap v82: 129 pgs: 20 active+undersized+degraded, 15 active+undersized, 94 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 125/1353 objects degraded (9.239%) 2024-02-19T21:56:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:35.677040+0000 mon.smithi131 (mon.0) 261 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:35.801370+0000 mon.smithi131 (mon.0) 262 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/1836168183,v1:172.21.15.131:6819/1836168183]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-19T21:56:37.006 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 21:56:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-2[85278]: debug 2024-02-19T21:56:36.690+0000 7f38d1729700 -1 osd.2 80 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-19T21:56:38.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:36.534520+0000 mgr.smithi131.cjubsp (mgr.14720) 118 : cluster [DBG] pgmap v83: 129 pgs: 20 active+undersized+degraded, 15 active+undersized, 94 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 237 B/s wr, 0 op/s; 125/1353 objects degraded (9.239%) 2024-02-19T21:56:38.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:36.678155+0000 mon.smithi131 (mon.0) 263 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/1836168183,v1:172.21.15.131:6819/1836168183]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-19T21:56:38.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:36.678269+0000 mon.smithi131 (mon.0) 264 : cluster [DBG] osdmap e83: 8 total, 7 up, 8 in 2024-02-19T21:56:38.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:36.679217+0000 mon.smithi131 (mon.0) 265 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/1836168183,v1:172.21.15.131:6819/1836168183]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:56:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:36.534520+0000 mgr.smithi131.cjubsp (mgr.14720) 118 : cluster [DBG] pgmap v83: 129 pgs: 20 active+undersized+degraded, 15 active+undersized, 94 active+clean; 303 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 237 B/s wr, 0 op/s; 125/1353 objects degraded (9.239%) 2024-02-19T21:56:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:36.678155+0000 mon.smithi131 (mon.0) 263 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/1836168183,v1:172.21.15.131:6819/1836168183]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-19T21:56:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:36.678269+0000 mon.smithi131 (mon.0) 264 : cluster [DBG] osdmap e83: 8 total, 7 up, 8 in 2024-02-19T21:56:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:36.679217+0000 mon.smithi131 (mon.0) 265 : audit [INF] from='osd.2 [v2:172.21.15.131:6818/1836168183,v1:172.21.15.131:6819/1836168183]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:56:39.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:37.676270+0000 mon.smithi131 (mon.0) 266 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:56:39.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:37.680606+0000 mon.smithi131 (mon.0) 267 : cluster [INF] osd.2 [v2:172.21.15.131:6818/1836168183,v1:172.21.15.131:6819/1836168183] boot 2024-02-19T21:56:39.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:37.680663+0000 mon.smithi131 (mon.0) 268 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-02-19T21:56:39.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:37.684803+0000 mon.smithi131 (mon.0) 269 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:56:39.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:37.805847+0000 mon.smithi131 (mon.0) 270 : cluster [WRN] Health check update: Degraded data redundancy: 125/1353 objects degraded (9.239%), 20 pgs degraded (PG_DEGRADED) 2024-02-19T21:56:39.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:38.682372+0000 mon.smithi131 (mon.0) 271 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-02-19T21:56:39.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:37.676270+0000 mon.smithi131 (mon.0) 266 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:56:39.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:37.680606+0000 mon.smithi131 (mon.0) 267 : cluster [INF] osd.2 [v2:172.21.15.131:6818/1836168183,v1:172.21.15.131:6819/1836168183] boot 2024-02-19T21:56:39.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:37.680663+0000 mon.smithi131 (mon.0) 268 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-02-19T21:56:39.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:37.684803+0000 mon.smithi131 (mon.0) 269 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-19T21:56:39.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:37.805847+0000 mon.smithi131 (mon.0) 270 : cluster [WRN] Health check update: Degraded data redundancy: 125/1353 objects degraded (9.239%), 20 pgs degraded (PG_DEGRADED) 2024-02-19T21:56:39.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:38.682372+0000 mon.smithi131 (mon.0) 271 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-02-19T21:56:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:38.534984+0000 mgr.smithi131.cjubsp (mgr.14720) 119 : cluster [DBG] pgmap v86: 129 pgs: 20 active+undersized+degraded, 15 active+undersized, 94 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 125/1353 objects degraded (9.239%) 2024-02-19T21:56:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:38.877563+0000 mon.smithi131 (mon.0) 272 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.162182+0000 mon.smithi131 (mon.0) 273 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:56:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.162776+0000 mon.smithi131 (mon.0) 274 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:56:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.166773+0000 mon.smithi131 (mon.0) 275 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.171526+0000 mon.smithi131 (mon.0) 276 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.174283+0000 mon.smithi131 (mon.0) 277 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:56:40.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.176162+0000 mon.smithi131 (mon.0) 278 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:40.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.177222+0000 mon.smithi131 (mon.0) 279 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:40.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.178272+0000 mon.smithi131 (mon.0) 280 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:40.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.179323+0000 mon.smithi131 (mon.0) 281 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-19T21:56:40.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:39.179443+0000 mgr.smithi131.cjubsp (mgr.14720) 120 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-19T21:56:40.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:56:39.180035+0000 mgr.smithi131.cjubsp (mgr.14720) 121 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-02-19T21:56:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:38.534984+0000 mgr.smithi131.cjubsp (mgr.14720) 119 : cluster [DBG] pgmap v86: 129 pgs: 20 active+undersized+degraded, 15 active+undersized, 94 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 125/1353 objects degraded (9.239%) 2024-02-19T21:56:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:38.877563+0000 mon.smithi131 (mon.0) 272 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.162182+0000 mon.smithi131 (mon.0) 273 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:56:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.162776+0000 mon.smithi131 (mon.0) 274 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:56:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.166773+0000 mon.smithi131 (mon.0) 275 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.171526+0000 mon.smithi131 (mon.0) 276 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.174283+0000 mon.smithi131 (mon.0) 277 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:56:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.176162+0000 mon.smithi131 (mon.0) 278 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:40.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.177222+0000 mon.smithi131 (mon.0) 279 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:40.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.178272+0000 mon.smithi131 (mon.0) 280 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:40.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.179323+0000 mon.smithi131 (mon.0) 281 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-19T21:56:40.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:39.179443+0000 mgr.smithi131.cjubsp (mgr.14720) 120 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-19T21:56:40.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:56:39.180035+0000 mgr.smithi131.cjubsp (mgr.14720) 121 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-02-19T21:56:40.922 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:42.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:40.535548+0000 mgr.smithi131.cjubsp (mgr.14720) 122 : cluster [DBG] pgmap v88: 129 pgs: 18 active+undersized+degraded, 13 active+undersized, 98 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 110/1353 objects degraded (8.130%) 2024-02-19T21:56:42.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:40.756798+0000 mon.smithi131 (mon.0) 282 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:40.535548+0000 mgr.smithi131.cjubsp (mgr.14720) 122 : cluster [DBG] pgmap v88: 129 pgs: 18 active+undersized+degraded, 13 active+undersized, 98 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 110/1353 objects degraded (8.130%) 2024-02-19T21:56:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:40.756798+0000 mon.smithi131 (mon.0) 282 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:43.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:42.756504+0000 mon.smithi131 (mon.0) 283 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 110/1353 objects degraded (8.130%), 18 pgs degraded) 2024-02-19T21:56:43.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:42.756534+0000 mon.smithi131 (mon.0) 284 : cluster [INF] Cluster is now healthy 2024-02-19T21:56:43.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:42.756504+0000 mon.smithi131 (mon.0) 283 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 110/1353 objects degraded (8.130%), 18 pgs degraded) 2024-02-19T21:56:43.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:42.756534+0000 mon.smithi131 (mon.0) 284 : cluster [INF] Cluster is now healthy 2024-02-19T21:56:44.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:42.536344+0000 mgr.smithi131.cjubsp (mgr.14720) 123 : cluster [DBG] pgmap v89: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:42.536344+0000 mgr.smithi131.cjubsp (mgr.14720) 123 : cluster [DBG] pgmap v89: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:45.963 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:46.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:44.536916+0000 mgr.smithi131.cjubsp (mgr.14720) 124 : cluster [DBG] pgmap v90: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 130 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:46.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:44.536916+0000 mgr.smithi131.cjubsp (mgr.14720) 124 : cluster [DBG] pgmap v90: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 130 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:48.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:46.537685+0000 mgr.smithi131.cjubsp (mgr.14720) 125 : cluster [DBG] pgmap v91: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 231 B/s rd, 115 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:48.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:46.537685+0000 mgr.smithi131.cjubsp (mgr.14720) 125 : cluster [DBG] pgmap v91: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 231 B/s rd, 115 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:48.538166+0000 mgr.smithi131.cjubsp (mgr.14720) 126 : cluster [DBG] pgmap v92: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:50.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:48.538166+0000 mgr.smithi131.cjubsp (mgr.14720) 126 : cluster [DBG] pgmap v92: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:50.355 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:56:50.932 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:56:50.932 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (9m) 12s ago 11m 21.1M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:56:50.932 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (104s) 12s ago 11m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (99s) 94s ago 10m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (9m) 12s ago 11m 34.6M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (5m) 12s ago 5m 39.0M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (5m) 94s ago 5m 13.2M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (2m) 12s ago 12m 438M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (2m) 94s ago 10m 387M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (2m) 12s ago 12m 44.9M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (115s) 94s ago 10m 26.9M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:56:50.933 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (3m) 12s ago 3m 79.8M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:56:50.934 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (3m) 94s ago 3m 49.5M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:56:50.934 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (10m) 12s ago 10m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:56:50.934 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (10m) 94s ago 9m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:56:50.934 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (82s) 12s ago 9m 112M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T21:56:50.934 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (50s) 12s ago 9m 100M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T21:56:50.934 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (19s) 12s ago 8m 11.6M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T21:56:50.934 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (8m) 12s ago 8m 342M 1944M 16.2.5 6933c2a0b7dd 9416ca1ad85f 2024-02-19T21:56:50.934 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (8m) 94s ago 8m 356M 2712M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:56:50.935 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (7m) 94s ago 7m 375M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:56:50.935 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (7m) 94s ago 7m 311M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:56:50.935 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (7m) 94s ago 7m 467M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:56:50.935 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (2m) 12s ago 10m 60.4M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:56:51.016 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:51.277 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 3, 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 5 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:56:51.278 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:56:51.279 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 7, 2024-02-19T21:56:51.279 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 7 2024-02-19T21:56:51.279 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:56:51.279 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:56:51.572 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:56:51.572 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:56:51.572 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:56:51.572 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:56:51.572 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:56:51.573 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T21:56:51.573 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:56:51.573 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:56:51.573 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:56:51.573 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "9/23 daemons upgraded", 2024-02-19T21:56:51.573 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading osd daemons", 2024-02-19T21:56:51.573 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:56:51.574 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:56:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:50.342661+0000 mgr.smithi131.cjubsp (mgr.14720) 127 : audit [DBG] from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:50.538593+0000 mgr.smithi131.cjubsp (mgr.14720) 128 : cluster [DBG] pgmap v93: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 172 B/s rd, 172 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:50.630057+0000 mgr.smithi131.cjubsp (mgr.14720) 129 : audit [DBG] from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:50.676171+0000 mon.smithi131 (mon.0) 285 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:50.768566+0000 mon.smithi131 (mon.0) 286 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:50.924090+0000 mgr.smithi131.cjubsp (mgr.14720) 130 : audit [DBG] from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:51.278635+0000 mon.smithi131 (mon.0) 287 : audit [DBG] from='client.? 172.21.15.131:0/150484926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:50.342661+0000 mgr.smithi131.cjubsp (mgr.14720) 127 : audit [DBG] from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:50.538593+0000 mgr.smithi131.cjubsp (mgr.14720) 128 : cluster [DBG] pgmap v93: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 172 B/s rd, 172 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:50.630057+0000 mgr.smithi131.cjubsp (mgr.14720) 129 : audit [DBG] from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:50.676171+0000 mon.smithi131 (mon.0) 285 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:50.768566+0000 mon.smithi131 (mon.0) 286 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:50.924090+0000 mgr.smithi131.cjubsp (mgr.14720) 130 : audit [DBG] from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:52.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:51.278635+0000 mon.smithi131 (mon.0) 287 : audit [DBG] from='client.? 172.21.15.131:0/150484926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:56:52.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:51.572494+0000 mgr.smithi131.cjubsp (mgr.14720) 131 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:51.572494+0000 mgr.smithi131.cjubsp (mgr.14720) 131 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:56:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:52.539471+0000 mgr.smithi131.cjubsp (mgr.14720) 132 : cluster [DBG] pgmap v94: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:52.539471+0000 mgr.smithi131.cjubsp (mgr.14720) 132 : cluster [DBG] pgmap v94: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:56:54.992 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:54.193955+0000 mon.smithi131 (mon.0) 288 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-19T21:56:54.993 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:54.194204+0000 mgr.smithi131.cjubsp (mgr.14720) 133 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-19T21:56:54.993 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:56:54.195433+0000 mgr.smithi131.cjubsp (mgr.14720) 134 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-02-19T21:56:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:54.193955+0000 mon.smithi131 (mon.0) 288 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-19T21:56:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:54.194204+0000 mgr.smithi131.cjubsp (mgr.14720) 133 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-19T21:56:55.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:56:54.195433+0000 mgr.smithi131.cjubsp (mgr.14720) 134 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-02-19T21:56:56.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:54.539981+0000 mgr.smithi131.cjubsp (mgr.14720) 135 : cluster [DBG] pgmap v95: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:56:56.048 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:56:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:54.539981+0000 mgr.smithi131.cjubsp (mgr.14720) 135 : cluster [DBG] pgmap v95: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:56:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:56:55.702692+0000 mgr.smithi131.cjubsp (mgr.14720) 136 : cephadm [INF] Upgrade: Updating osd.3 2024-02-19T21:56:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:55.706668+0000 mon.smithi131 (mon.0) 289 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:55.707320+0000 mon.smithi131 (mon.0) 290 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-19T21:56:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:56:55.707815+0000 mon.smithi131 (mon.0) 291 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:56:57.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:56:55.708797+0000 mgr.smithi131.cjubsp (mgr.14720) 137 : cephadm [INF] Deploying daemon osd.3 on smithi131 2024-02-19T21:56:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:56:55.702692+0000 mgr.smithi131.cjubsp (mgr.14720) 136 : cephadm [INF] Upgrade: Updating osd.3 2024-02-19T21:56:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:55.706668+0000 mon.smithi131 (mon.0) 289 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:56:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:55.707320+0000 mon.smithi131 (mon.0) 290 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-19T21:56:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:56:55.707815+0000 mon.smithi131 (mon.0) 291 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:56:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:56:55.708797+0000 mgr.smithi131.cjubsp (mgr.14720) 137 : cephadm [INF] Deploying daemon osd.3 on smithi131 2024-02-19T21:56:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:56.540459+0000 mgr.smithi131.cjubsp (mgr.14720) 138 : cluster [DBG] pgmap v96: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:56:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:56.540459+0000 mgr.smithi131.cjubsp (mgr.14720) 138 : cluster [DBG] pgmap v96: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:56:59.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:56:58 smithi131 systemd[1]: Stopping Ceph osd.3 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:56:59.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3[54751]: debug 2024-02-19T21:56:59.056+0000 7fb961604700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T21:56:59.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3[54751]: debug 2024-02-19T21:56:59.056+0000 7fb961604700 -1 osd.3 85 *** Got signal Terminated *** 2024-02-19T21:56:59.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3[54751]: debug 2024-02-19T21:56:59.056+0000 7fb961604700 -1 osd.3 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T21:56:59.725 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:56:59 smithi131 bash[87150]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3 2024-02-19T21:56:59.990 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:56:59 smithi131 bash[87204]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3 2024-02-19T21:56:59.990 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:58.540968+0000 mgr.smithi131.cjubsp (mgr.14720) 139 : cluster [DBG] pgmap v97: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332106+0000 mon.smithi131 (mon.0) 292 : cluster [DBG] osd.3 reported immediately failed by osd.0 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332156+0000 mon.smithi131 (mon.0) 293 : cluster [INF] osd.3 failed (root=default,host=smithi131) (connection refused reported by osd.0) 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332275+0000 mon.smithi131 (mon.0) 294 : cluster [DBG] osd.3 reported immediately failed by osd.0 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332366+0000 mon.smithi131 (mon.0) 295 : cluster [DBG] osd.3 reported immediately failed by osd.1 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332463+0000 mon.smithi131 (mon.0) 296 : cluster [DBG] osd.3 reported immediately failed by osd.2 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332535+0000 mon.smithi131 (mon.0) 297 : cluster [DBG] osd.3 reported immediately failed by osd.6 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332612+0000 mon.smithi131 (mon.0) 298 : cluster [DBG] osd.3 reported immediately failed by osd.5 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332700+0000 mon.smithi131 (mon.0) 299 : cluster [DBG] osd.3 reported immediately failed by osd.7 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332767+0000 mon.smithi131 (mon.0) 300 : cluster [DBG] osd.3 reported immediately failed by osd.4 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332866+0000 mon.smithi131 (mon.0) 301 : cluster [DBG] osd.3 reported immediately failed by osd.1 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.332907+0000 mon.smithi131 (mon.0) 302 : cluster [DBG] osd.3 reported immediately failed by osd.2 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.333000+0000 mon.smithi131 (mon.0) 303 : cluster [DBG] osd.3 reported immediately failed by osd.6 2024-02-19T21:56:59.991 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.333038+0000 mon.smithi131 (mon.0) 304 : cluster [DBG] osd.3 reported immediately failed by osd.5 2024-02-19T21:56:59.992 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.333148+0000 mon.smithi131 (mon.0) 305 : cluster [DBG] osd.3 reported immediately failed by osd.7 2024-02-19T21:56:59.992 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:56:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.333226+0000 mon.smithi131 (mon.0) 306 : cluster [DBG] osd.3 reported immediately failed by osd.4 2024-02-19T21:57:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:58.540968+0000 mgr.smithi131.cjubsp (mgr.14720) 139 : cluster [DBG] pgmap v97: 129 pgs: 129 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:57:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332106+0000 mon.smithi131 (mon.0) 292 : cluster [DBG] osd.3 reported immediately failed by osd.0 2024-02-19T21:57:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332156+0000 mon.smithi131 (mon.0) 293 : cluster [INF] osd.3 failed (root=default,host=smithi131) (connection refused reported by osd.0) 2024-02-19T21:57:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332275+0000 mon.smithi131 (mon.0) 294 : cluster [DBG] osd.3 reported immediately failed by osd.0 2024-02-19T21:57:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332366+0000 mon.smithi131 (mon.0) 295 : cluster [DBG] osd.3 reported immediately failed by osd.1 2024-02-19T21:57:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332463+0000 mon.smithi131 (mon.0) 296 : cluster [DBG] osd.3 reported immediately failed by osd.2 2024-02-19T21:57:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332535+0000 mon.smithi131 (mon.0) 297 : cluster [DBG] osd.3 reported immediately failed by osd.6 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332612+0000 mon.smithi131 (mon.0) 298 : cluster [DBG] osd.3 reported immediately failed by osd.5 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332700+0000 mon.smithi131 (mon.0) 299 : cluster [DBG] osd.3 reported immediately failed by osd.7 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332767+0000 mon.smithi131 (mon.0) 300 : cluster [DBG] osd.3 reported immediately failed by osd.4 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332866+0000 mon.smithi131 (mon.0) 301 : cluster [DBG] osd.3 reported immediately failed by osd.1 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.332907+0000 mon.smithi131 (mon.0) 302 : cluster [DBG] osd.3 reported immediately failed by osd.2 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.333000+0000 mon.smithi131 (mon.0) 303 : cluster [DBG] osd.3 reported immediately failed by osd.6 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.333038+0000 mon.smithi131 (mon.0) 304 : cluster [DBG] osd.3 reported immediately failed by osd.5 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.333148+0000 mon.smithi131 (mon.0) 305 : cluster [DBG] osd.3 reported immediately failed by osd.7 2024-02-19T21:57:00.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:56:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.333226+0000 mon.smithi131 (mon.0) 306 : cluster [DBG] osd.3 reported immediately failed by osd.4 2024-02-19T21:57:01.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.714332+0000 mon.smithi131 (mon.0) 307 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:57:01.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:56:59.720538+0000 mon.smithi131 (mon.0) 308 : cluster [DBG] osdmap e86: 8 total, 7 up, 8 in 2024-02-19T21:57:01.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:00.719368+0000 mon.smithi131 (mon.0) 309 : cluster [DBG] osdmap e87: 8 total, 7 up, 8 in 2024-02-19T21:57:01.105 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.714332+0000 mon.smithi131 (mon.0) 307 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:57:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:56:59.720538+0000 mon.smithi131 (mon.0) 308 : cluster [DBG] osdmap e86: 8 total, 7 up, 8 in 2024-02-19T21:57:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:00.719368+0000 mon.smithi131 (mon.0) 309 : cluster [DBG] osdmap e87: 8 total, 7 up, 8 in 2024-02-19T21:57:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:00.541382+0000 mgr.smithi131.cjubsp (mgr.14720) 140 : cluster [DBG] pgmap v99: 129 pgs: 20 stale+active+clean, 109 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:57:02.006 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:01 smithi131 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.3.service: Succeeded. 2024-02-19T21:57:02.007 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:01 smithi131 systemd[1]: Stopped Ceph osd.3 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:57:02.007 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:01 smithi131 systemd[1]: Starting Ceph osd.3 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:57:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:00.541382+0000 mgr.smithi131.cjubsp (mgr.14720) 140 : cluster [DBG] pgmap v99: 129 pgs: 20 stale+active+clean, 109 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:57:02.756 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:02 smithi131 bash[87486]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-02-19T21:57:02.757 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:02 smithi131 bash[87486]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-19T21:57:02.757 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:02 smithi131 bash[87486]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-02-19T21:57:02.757 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:02 smithi131 bash[87486]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-02-19T21:57:02.757 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:02 smithi131 bash[87486]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-02-19T21:57:02.757 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:02 smithi131 bash[87486]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-19T21:57:02.758 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:02 smithi131 bash[87486]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-19T21:57:02.758 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:02 smithi131 bash[87486]: --> ceph-volume lvm activate successful for osd ID: 3 2024-02-19T21:57:03.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:02.745183+0000 mon.smithi131 (mon.0) 310 : cluster [WRN] Health check failed: Degraded data redundancy: 112/1353 objects degraded (8.278%), 23 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:03.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:02.745183+0000 mon.smithi131 (mon.0) 310 : cluster [WRN] Health check failed: Degraded data redundancy: 112/1353 objects degraded (8.278%), 23 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:02.542010+0000 mgr.smithi131.cjubsp (mgr.14720) 141 : cluster [DBG] pgmap v101: 129 pgs: 23 active+undersized+degraded, 4 stale+active+clean, 17 active+undersized, 85 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 112/1353 objects degraded (8.278%) 2024-02-19T21:57:04.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:02.542010+0000 mgr.smithi131.cjubsp (mgr.14720) 141 : cluster [DBG] pgmap v101: 129 pgs: 23 active+undersized+degraded, 4 stale+active+clean, 17 active+undersized, 85 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 112/1353 objects degraded (8.278%) 2024-02-19T21:57:04.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-3[87743]: debug 2024-02-19T21:57:03.824+0000 7fe8da406700 1 -- 172.21.15.131:0/857836783 <== mon.0 v2:172.21.15.131:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f7711f0680 con 0x55f7711b7000 2024-02-19T21:57:04.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:03 smithi131 bash[87728]: 840766dde496c69866ae5f3a565a8e47bc9c1d23c4ef880d3ac509f28062c276 2024-02-19T21:57:04.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:03 smithi131 systemd[1]: Started Ceph osd.3 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:57:04.756 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-3[87743]: debug 2024-02-19T21:57:04.349+0000 7fe8e1e94380 -1 Falling back to public interface 2024-02-19T21:57:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:03.963605+0000 mon.smithi131 (mon.0) 311 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:03.964391+0000 mon.smithi131 (mon.0) 312 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:57:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:03.963605+0000 mon.smithi131 (mon.0) 311 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:03.964391+0000 mon.smithi131 (mon.0) 312 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:57:06.142 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:06.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:04.542427+0000 mgr.smithi131.cjubsp (mgr.14720) 142 : cluster [DBG] pgmap v102: 129 pgs: 23 active+undersized+degraded, 4 stale+active+clean, 17 active+undersized, 85 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 112/1353 objects degraded (8.278%) 2024-02-19T21:57:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:05.676582+0000 mon.smithi131 (mon.0) 313 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:04.542427+0000 mgr.smithi131.cjubsp (mgr.14720) 142 : cluster [DBG] pgmap v102: 129 pgs: 23 active+undersized+degraded, 4 stale+active+clean, 17 active+undersized, 85 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 112/1353 objects degraded (8.278%) 2024-02-19T21:57:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:05.676582+0000 mon.smithi131 (mon.0) 313 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:06.543472+0000 mgr.smithi131.cjubsp (mgr.14720) 143 : cluster [DBG] pgmap v103: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 159/1353 objects degraded (11.752%) 2024-02-19T21:57:08.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:07.810965+0000 mon.smithi131 (mon.0) 314 : cluster [WRN] Health check update: Degraded data redundancy: 159/1353 objects degraded (11.752%), 30 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:06.543472+0000 mgr.smithi131.cjubsp (mgr.14720) 143 : cluster [DBG] pgmap v103: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 159/1353 objects degraded (11.752%) 2024-02-19T21:57:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:07.810965+0000 mon.smithi131 (mon.0) 314 : cluster [WRN] Health check update: Degraded data redundancy: 159/1353 objects degraded (11.752%), 30 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:08.981 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-3[87743]: debug 2024-02-19T21:57:08.615+0000 7fe8e1e94380 -1 osd.3 85 log_to_monitors {default=true} 2024-02-19T21:57:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:08.620616+0000 mon.smithi131 (mon.0) 315 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/2691683030,v1:172.21.15.131:6827/2691683030]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-19T21:57:09.256 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 21:57:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-3[87743]: debug 2024-02-19T21:57:08.994+0000 7fe8d0b76700 -1 osd.3 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-19T21:57:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:08.620616+0000 mon.smithi131 (mon.0) 315 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/2691683030,v1:172.21.15.131:6827/2691683030]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-19T21:57:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:08.543969+0000 mgr.smithi131.cjubsp (mgr.14720) 144 : cluster [DBG] pgmap v104: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 232 B/s wr, 0 op/s; 159/1353 objects degraded (11.752%) 2024-02-19T21:57:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:08.985186+0000 mon.smithi131 (mon.0) 316 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/2691683030,v1:172.21.15.131:6827/2691683030]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-19T21:57:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:08.985244+0000 mon.smithi131 (mon.0) 317 : cluster [DBG] osdmap e88: 8 total, 7 up, 8 in 2024-02-19T21:57:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:08.985756+0000 mon.smithi131 (mon.0) 318 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/2691683030,v1:172.21.15.131:6827/2691683030]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:57:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:08.543969+0000 mgr.smithi131.cjubsp (mgr.14720) 144 : cluster [DBG] pgmap v104: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 303 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 232 B/s wr, 0 op/s; 159/1353 objects degraded (11.752%) 2024-02-19T21:57:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:08.985186+0000 mon.smithi131 (mon.0) 316 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/2691683030,v1:172.21.15.131:6827/2691683030]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-19T21:57:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:08.985244+0000 mon.smithi131 (mon.0) 317 : cluster [DBG] osdmap e88: 8 total, 7 up, 8 in 2024-02-19T21:57:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:08.985756+0000 mon.smithi131 (mon.0) 318 : audit [INF] from='osd.3 [v2:172.21.15.131:6826/2691683030,v1:172.21.15.131:6827/2691683030]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi131", "root=default"]}]: dispatch 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:09.980504+0000 mon.smithi131 (mon.0) 319 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:09.983957+0000 mon.smithi131 (mon.0) 320 : cluster [INF] osd.3 [v2:172.21.15.131:6826/2691683030,v1:172.21.15.131:6827/2691683030] boot 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:09.984003+0000 mon.smithi131 (mon.0) 321 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:09.984448+0000 mon.smithi131 (mon.0) 322 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.476930+0000 mon.smithi131 (mon.0) 323 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.477678+0000 mon.smithi131 (mon.0) 324 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.478304+0000 mon.smithi131 (mon.0) 325 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.482095+0000 mon.smithi131 (mon.0) 326 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.487375+0000 mon.smithi131 (mon.0) 327 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:11.001 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.490023+0000 mon.smithi131 (mon.0) 328 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:57:11.002 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.491916+0000 mon.smithi131 (mon.0) 329 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:11.002 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.492912+0000 mon.smithi131 (mon.0) 330 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:11.002 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.493899+0000 mon.smithi131 (mon.0) 331 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:11.002 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.495024+0000 mon.smithi131 (mon.0) 332 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-19T21:57:11.002 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:10.986747+0000 mon.smithi131 (mon.0) 333 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-02-19T21:57:11.182 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:09.980504+0000 mon.smithi131 (mon.0) 319 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:57:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:09.983957+0000 mon.smithi131 (mon.0) 320 : cluster [INF] osd.3 [v2:172.21.15.131:6826/2691683030,v1:172.21.15.131:6827/2691683030] boot 2024-02-19T21:57:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:09.984003+0000 mon.smithi131 (mon.0) 321 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-02-19T21:57:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:09.984448+0000 mon.smithi131 (mon.0) 322 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-19T21:57:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.476930+0000 mon.smithi131 (mon.0) 323 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.477678+0000 mon.smithi131 (mon.0) 324 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:57:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.478304+0000 mon.smithi131 (mon.0) 325 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:57:11.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.482095+0000 mon.smithi131 (mon.0) 326 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:11.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.487375+0000 mon.smithi131 (mon.0) 327 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:11.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.490023+0000 mon.smithi131 (mon.0) 328 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:57:11.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.491916+0000 mon.smithi131 (mon.0) 329 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:11.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.492912+0000 mon.smithi131 (mon.0) 330 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:11.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.493899+0000 mon.smithi131 (mon.0) 331 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:11.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.495024+0000 mon.smithi131 (mon.0) 332 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-19T21:57:11.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:10.986747+0000 mon.smithi131 (mon.0) 333 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-02-19T21:57:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:10.495150+0000 mgr.smithi131.cjubsp (mgr.14720) 145 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-19T21:57:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:57:10.495708+0000 mgr.smithi131.cjubsp (mgr.14720) 146 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-02-19T21:57:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:10.544599+0000 mgr.smithi131.cjubsp (mgr.14720) 147 : cluster [DBG] pgmap v107: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 159/1353 objects degraded (11.752%) 2024-02-19T21:57:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:10.495150+0000 mgr.smithi131.cjubsp (mgr.14720) 145 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-19T21:57:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:57:10.495708+0000 mgr.smithi131.cjubsp (mgr.14720) 146 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-02-19T21:57:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:10.544599+0000 mgr.smithi131.cjubsp (mgr.14720) 147 : cluster [DBG] pgmap v107: 129 pgs: 30 active+undersized+degraded, 20 active+undersized, 79 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 159/1353 objects degraded (11.752%) 2024-02-19T21:57:13.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:12.998955+0000 mon.smithi131 (mon.0) 334 : cluster [WRN] Health check update: Degraded data redundancy: 89/1353 objects degraded (6.578%), 18 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:13.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:12.998955+0000 mon.smithi131 (mon.0) 334 : cluster [WRN] Health check update: Degraded data redundancy: 89/1353 objects degraded (6.578%), 18 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:14.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:12.545369+0000 mgr.smithi131.cjubsp (mgr.14720) 148 : cluster [DBG] pgmap v109: 129 pgs: 7 peering, 18 active+undersized+degraded, 8 active+undersized, 96 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s wr, 0 op/s; 89/1353 objects degraded (6.578%); 0 B/s, 0 objects/s recovering 2024-02-19T21:57:14.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:12.545369+0000 mgr.smithi131.cjubsp (mgr.14720) 148 : cluster [DBG] pgmap v109: 129 pgs: 7 peering, 18 active+undersized+degraded, 8 active+undersized, 96 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s wr, 0 op/s; 89/1353 objects degraded (6.578%); 0 B/s, 0 objects/s recovering 2024-02-19T21:57:16.230 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:16.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:14.545937+0000 mgr.smithi131.cjubsp (mgr.14720) 149 : cluster [DBG] pgmap v110: 129 pgs: 7 peering, 4 active+undersized+degraded, 2 active+undersized, 116 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 170 B/s wr, 5 op/s; 27/1353 objects degraded (1.996%); 0 B/s, 0 objects/s recovering 2024-02-19T21:57:16.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:15.785150+0000 mon.smithi131 (mon.0) 335 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:16.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:14.545937+0000 mgr.smithi131.cjubsp (mgr.14720) 149 : cluster [DBG] pgmap v110: 129 pgs: 7 peering, 4 active+undersized+degraded, 2 active+undersized, 116 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 170 B/s wr, 5 op/s; 27/1353 objects degraded (1.996%); 0 B/s, 0 objects/s recovering 2024-02-19T21:57:16.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:15.785150+0000 mon.smithi131 (mon.0) 335 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:17.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:17.057691+0000 mon.smithi131 (mon.0) 336 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/1353 objects degraded (1.996%), 4 pgs degraded) 2024-02-19T21:57:17.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:17.057748+0000 mon.smithi131 (mon.0) 337 : cluster [INF] Cluster is now healthy 2024-02-19T21:57:17.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:17.057691+0000 mon.smithi131 (mon.0) 336 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/1353 objects degraded (1.996%), 4 pgs degraded) 2024-02-19T21:57:17.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:17.057748+0000 mon.smithi131 (mon.0) 337 : cluster [INF] Cluster is now healthy 2024-02-19T21:57:18.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:16.546527+0000 mgr.smithi131.cjubsp (mgr.14720) 150 : cluster [DBG] pgmap v111: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 135 B/s wr, 4 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:18.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:16.546527+0000 mgr.smithi131.cjubsp (mgr.14720) 150 : cluster [DBG] pgmap v111: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 135 B/s wr, 4 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:20.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:18.547115+0000 mgr.smithi131.cjubsp (mgr.14720) 151 : cluster [DBG] pgmap v112: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 239 B/s wr, 4 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:20.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:18.547115+0000 mgr.smithi131.cjubsp (mgr.14720) 151 : cluster [DBG] pgmap v112: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 239 B/s wr, 4 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:21.267 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:21.888 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:57:22.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:20.547632+0000 mgr.smithi131.cjubsp (mgr.14720) 152 : cluster [DBG] pgmap v113: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 204 B/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:22.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:20.679336+0000 mon.smithi131 (mon.0) 338 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:20.547632+0000 mgr.smithi131.cjubsp (mgr.14720) 152 : cluster [DBG] pgmap v113: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 204 B/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:20.679336+0000 mon.smithi131 (mon.0) 338 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:22.458 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (10m) 11s ago 11m 21.1M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (2m) 11s ago 11m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (2m) 2m ago 10m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (10m) 11s ago 11m 34.6M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (6m) 11s ago 6m 39.0M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (6m) 2m ago 6m 13.2M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (3m) 11s ago 13m 440M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (2m) 2m ago 10m 387M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (2m) 11s ago 13m 48.0M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (2m) 2m ago 10m 26.9M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (3m) 11s ago 3m 80.0M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (3m) 2m ago 3m 49.5M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (11m) 11s ago 11m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (10m) 2m ago 10m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (114s) 11s ago 10m 113M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T21:57:22.459 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (82s) 11s ago 9m 101M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T21:57:22.460 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (51s) 11s ago 9m 80.6M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T21:57:22.460 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (18s) 11s ago 9m 13.3M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T21:57:22.460 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (8m) 2m ago 8m 356M 2712M 16.2.5 6933c2a0b7dd 5271cc3ea031 2024-02-19T21:57:22.460 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (8m) 2m ago 8m 375M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:57:22.460 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (8m) 2m ago 8m 311M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:57:22.460 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (7m) 2m ago 7m 467M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:57:22.460 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (3m) 11s ago 11m 60.5M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:57:22.801 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:57:22.801 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:57:22.801 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:57:22.801 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 4, 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 4 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 8, 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 6 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:57:22.802 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:57:22.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:21.876522+0000 mgr.smithi131.cjubsp (mgr.14720) 153 : audit [DBG] from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:22.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:22.166233+0000 mgr.smithi131.cjubsp (mgr.14720) 154 : audit [DBG] from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:23.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:21.876522+0000 mgr.smithi131.cjubsp (mgr.14720) 153 : audit [DBG] from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:23.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:22.166233+0000 mgr.smithi131.cjubsp (mgr.14720) 154 : audit [DBG] from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:23.092 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "10/23 daemons upgraded", 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading osd daemons", 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:57:23.093 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:57:24.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:22.450852+0000 mgr.smithi131.cjubsp (mgr.14720) 155 : audit [DBG] from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:24.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:22.548373+0000 mgr.smithi131.cjubsp (mgr.14720) 156 : cluster [DBG] pgmap v114: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 265 B/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:24.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:22.802564+0000 mon.smithi131 (mon.0) 339 : audit [DBG] from='client.? 172.21.15.131:0/927369936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:24.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:23.093113+0000 mgr.smithi131.cjubsp (mgr.14720) 157 : audit [DBG] from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:22.450852+0000 mgr.smithi131.cjubsp (mgr.14720) 155 : audit [DBG] from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:22.548373+0000 mgr.smithi131.cjubsp (mgr.14720) 156 : cluster [DBG] pgmap v114: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 265 B/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:24.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:22.802564+0000 mon.smithi131 (mon.0) 339 : audit [DBG] from='client.? 172.21.15.131:0/927369936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:24.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:23.093113+0000 mgr.smithi131.cjubsp (mgr.14720) 157 : audit [DBG] from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:25.941 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:24.548916+0000 mgr.smithi131.cjubsp (mgr.14720) 158 : cluster [DBG] pgmap v115: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 170 B/s wr, 3 op/s 2024-02-19T21:57:25.941 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:25.498980+0000 mon.smithi131 (mon.0) 340 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-19T21:57:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:24.548916+0000 mgr.smithi131.cjubsp (mgr.14720) 158 : cluster [DBG] pgmap v115: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 170 B/s wr, 3 op/s 2024-02-19T21:57:26.069 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:25.498980+0000 mon.smithi131 (mon.0) 340 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-19T21:57:26.307 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:25.499222+0000 mgr.smithi131.cjubsp (mgr.14720) 159 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-19T21:57:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:57:25.500430+0000 mgr.smithi131.cjubsp (mgr.14720) 160 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-02-19T21:57:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:25.796754+0000 mon.smithi131 (mon.0) 341 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:25.499222+0000 mgr.smithi131.cjubsp (mgr.14720) 159 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-19T21:57:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:57:25.500430+0000 mgr.smithi131.cjubsp (mgr.14720) 160 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-02-19T21:57:27.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:25.796754+0000 mon.smithi131 (mon.0) 341 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:26.549565+0000 mgr.smithi131.cjubsp (mgr.14720) 161 : cluster [DBG] pgmap v116: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:57:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:57:27.023611+0000 mgr.smithi131.cjubsp (mgr.14720) 162 : cephadm [INF] Upgrade: Updating osd.4 2024-02-19T21:57:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:27.028641+0000 mon.smithi131 (mon.0) 342 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:27.029576+0000 mon.smithi131 (mon.0) 343 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-19T21:57:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:27.030241+0000 mon.smithi131 (mon.0) 344 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:57:28.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:57:27.031502+0000 mgr.smithi131.cjubsp (mgr.14720) 163 : cephadm [INF] Deploying daemon osd.4 on smithi196 2024-02-19T21:57:28.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:26.549565+0000 mgr.smithi131.cjubsp (mgr.14720) 161 : cluster [DBG] pgmap v116: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:57:28.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:57:27.023611+0000 mgr.smithi131.cjubsp (mgr.14720) 162 : cephadm [INF] Upgrade: Updating osd.4 2024-02-19T21:57:28.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:27.028641+0000 mon.smithi131 (mon.0) 342 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:28.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:27.029576+0000 mon.smithi131 (mon.0) 343 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-19T21:57:28.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:27.030241+0000 mon.smithi131 (mon.0) 344 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:57:28.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:57:27.031502+0000 mgr.smithi131.cjubsp (mgr.14720) 163 : cephadm [INF] Deploying daemon osd.4 on smithi196 2024-02-19T21:57:30.227 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:28.550022+0000 mgr.smithi131.cjubsp (mgr.14720) 164 : cluster [DBG] pgmap v117: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:57:30.227 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:30 smithi196 systemd[1]: Stopping Ceph osd.4 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:57:30.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:28.550022+0000 mgr.smithi131.cjubsp (mgr.14720) 164 : cluster [DBG] pgmap v117: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:57:30.590 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4[40548]: debug 2024-02-19T21:57:30.226+0000 7f9ba36dd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T21:57:30.591 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4[40548]: debug 2024-02-19T21:57:30.226+0000 7f9ba36dd700 -1 osd.4 90 *** Got signal Terminated *** 2024-02-19T21:57:30.591 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4[40548]: debug 2024-02-19T21:57:30.226+0000 7f9ba36dd700 -1 osd.4 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T21:57:30.923 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:30 smithi196 bash[59397]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4 2024-02-19T21:57:31.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:30 smithi196 bash[59452]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4" found: no such container 2024-02-19T21:57:31.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.503783+0000 mon.smithi131 (mon.0) 345 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.503847+0000 mon.smithi131 (mon.0) 346 : cluster [INF] osd.4 failed (root=default,host=smithi196) (connection refused reported by osd.7) 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504121+0000 mon.smithi131 (mon.0) 347 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504223+0000 mon.smithi131 (mon.0) 348 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504373+0000 mon.smithi131 (mon.0) 349 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504477+0000 mon.smithi131 (mon.0) 350 : cluster [DBG] osd.4 reported immediately failed by osd.0 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504570+0000 mon.smithi131 (mon.0) 351 : cluster [DBG] osd.4 reported immediately failed by osd.1 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504666+0000 mon.smithi131 (mon.0) 352 : cluster [DBG] osd.4 reported immediately failed by osd.2 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504757+0000 mon.smithi131 (mon.0) 353 : cluster [DBG] osd.4 reported immediately failed by osd.3 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504862+0000 mon.smithi131 (mon.0) 354 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.504954+0000 mon.smithi131 (mon.0) 355 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.505041+0000 mon.smithi131 (mon.0) 356 : cluster [DBG] osd.4 reported immediately failed by osd.0 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.505140+0000 mon.smithi131 (mon.0) 357 : cluster [DBG] osd.4 reported immediately failed by osd.1 2024-02-19T21:57:31.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.505230+0000 mon.smithi131 (mon.0) 358 : cluster [DBG] osd.4 reported immediately failed by osd.2 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.505349+0000 mon.smithi131 (mon.0) 359 : cluster [DBG] osd.4 reported immediately failed by osd.3 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.852482+0000 mon.smithi131 (mon.0) 360 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.855821+0000 mon.smithi131 (mon.0) 361 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.864256+0000 mon.smithi131 (mon.0) 362 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.904491+0000 mon.smithi131 (mon.0) 363 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.904596+0000 mon.smithi131 (mon.0) 364 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.904661+0000 mon.smithi131 (mon.0) 365 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.904739+0000 mon.smithi131 (mon.0) 366 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.904819+0000 mon.smithi131 (mon.0) 367 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.904944+0000 mon.smithi131 (mon.0) 368 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.905520+0000 mon.smithi131 (mon.0) 369 : cluster [DBG] osd.4 reported immediately failed by osd.0 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.905602+0000 mon.smithi131 (mon.0) 370 : cluster [DBG] osd.4 reported immediately failed by osd.1 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.905824+0000 mon.smithi131 (mon.0) 371 : cluster [DBG] osd.4 reported immediately failed by osd.2 2024-02-19T21:57:31.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.905970+0000 mon.smithi131 (mon.0) 372 : cluster [DBG] osd.4 reported immediately failed by osd.3 2024-02-19T21:57:31.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.906119+0000 mon.smithi131 (mon.0) 373 : cluster [DBG] osd.4 reported immediately failed by osd.0 2024-02-19T21:57:31.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.906248+0000 mon.smithi131 (mon.0) 374 : cluster [DBG] osd.4 reported immediately failed by osd.1 2024-02-19T21:57:31.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.906397+0000 mon.smithi131 (mon.0) 375 : cluster [DBG] osd.4 reported immediately failed by osd.2 2024-02-19T21:57:31.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.906527+0000 mon.smithi131 (mon.0) 376 : cluster [DBG] osd.4 reported immediately failed by osd.3 2024-02-19T21:57:31.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.503783+0000 mon.smithi131 (mon.0) 345 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.503847+0000 mon.smithi131 (mon.0) 346 : cluster [INF] osd.4 failed (root=default,host=smithi196) (connection refused reported by osd.7) 2024-02-19T21:57:31.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504121+0000 mon.smithi131 (mon.0) 347 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504223+0000 mon.smithi131 (mon.0) 348 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504373+0000 mon.smithi131 (mon.0) 349 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504477+0000 mon.smithi131 (mon.0) 350 : cluster [DBG] osd.4 reported immediately failed by osd.0 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504570+0000 mon.smithi131 (mon.0) 351 : cluster [DBG] osd.4 reported immediately failed by osd.1 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504666+0000 mon.smithi131 (mon.0) 352 : cluster [DBG] osd.4 reported immediately failed by osd.2 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504757+0000 mon.smithi131 (mon.0) 353 : cluster [DBG] osd.4 reported immediately failed by osd.3 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504862+0000 mon.smithi131 (mon.0) 354 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.504954+0000 mon.smithi131 (mon.0) 355 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.505041+0000 mon.smithi131 (mon.0) 356 : cluster [DBG] osd.4 reported immediately failed by osd.0 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.505140+0000 mon.smithi131 (mon.0) 357 : cluster [DBG] osd.4 reported immediately failed by osd.1 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.505230+0000 mon.smithi131 (mon.0) 358 : cluster [DBG] osd.4 reported immediately failed by osd.2 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.505349+0000 mon.smithi131 (mon.0) 359 : cluster [DBG] osd.4 reported immediately failed by osd.3 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.852482+0000 mon.smithi131 (mon.0) 360 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.855821+0000 mon.smithi131 (mon.0) 361 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.864256+0000 mon.smithi131 (mon.0) 362 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.904491+0000 mon.smithi131 (mon.0) 363 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.904596+0000 mon.smithi131 (mon.0) 364 : cluster [DBG] osd.4 reported immediately failed by osd.6 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.904661+0000 mon.smithi131 (mon.0) 365 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.904739+0000 mon.smithi131 (mon.0) 366 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.904819+0000 mon.smithi131 (mon.0) 367 : cluster [DBG] osd.4 reported immediately failed by osd.5 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.904944+0000 mon.smithi131 (mon.0) 368 : cluster [DBG] osd.4 reported immediately failed by osd.7 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.905520+0000 mon.smithi131 (mon.0) 369 : cluster [DBG] osd.4 reported immediately failed by osd.0 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.905602+0000 mon.smithi131 (mon.0) 370 : cluster [DBG] osd.4 reported immediately failed by osd.1 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.905824+0000 mon.smithi131 (mon.0) 371 : cluster [DBG] osd.4 reported immediately failed by osd.2 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.905970+0000 mon.smithi131 (mon.0) 372 : cluster [DBG] osd.4 reported immediately failed by osd.3 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.906119+0000 mon.smithi131 (mon.0) 373 : cluster [DBG] osd.4 reported immediately failed by osd.0 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.906248+0000 mon.smithi131 (mon.0) 374 : cluster [DBG] osd.4 reported immediately failed by osd.1 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.906397+0000 mon.smithi131 (mon.0) 375 : cluster [DBG] osd.4 reported immediately failed by osd.2 2024-02-19T21:57:31.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.906527+0000 mon.smithi131 (mon.0) 376 : cluster [DBG] osd.4 reported immediately failed by osd.3 2024-02-19T21:57:32.096 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:30.550569+0000 mgr.smithi131.cjubsp (mgr.14720) 165 : cluster [DBG] pgmap v118: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:57:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:31.030955+0000 mon.smithi131 (mon.0) 377 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:57:32.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:31.041666+0000 mon.smithi131 (mon.0) 378 : cluster [DBG] osdmap e91: 8 total, 7 up, 8 in 2024-02-19T21:57:32.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:30.550569+0000 mgr.smithi131.cjubsp (mgr.14720) 165 : cluster [DBG] pgmap v118: 129 pgs: 129 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:57:32.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:31.030955+0000 mon.smithi131 (mon.0) 377 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:57:32.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:31.041666+0000 mon.smithi131 (mon.0) 378 : cluster [DBG] osdmap e91: 8 total, 7 up, 8 in 2024-02-19T21:57:33.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:32.044945+0000 mon.smithi131 (mon.0) 379 : cluster [DBG] osdmap e92: 8 total, 7 up, 8 in 2024-02-19T21:57:33.173 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:32 smithi196 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.4.service: Succeeded. 2024-02-19T21:57:33.174 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:32 smithi196 systemd[1]: Stopped Ceph osd.4 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:57:33.174 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:32 smithi196 systemd[1]: Starting Ceph osd.4 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:57:33.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:32.044945+0000 mon.smithi131 (mon.0) 379 : cluster [DBG] osdmap e92: 8 total, 7 up, 8 in 2024-02-19T21:57:34.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:32.551145+0000 mgr.smithi131.cjubsp (mgr.14720) 166 : cluster [DBG] pgmap v121: 129 pgs: 30 peering, 9 stale+active+clean, 90 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-02-19T21:57:34.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:33.042192+0000 mon.smithi131 (mon.0) 380 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-02-19T21:57:34.174 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:33 smithi196 bash[59710]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-02-19T21:57:34.174 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:33 smithi196 bash[59710]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-19T21:57:34.174 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:33 smithi196 bash[59710]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-02-19T21:57:34.174 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:33 smithi196 bash[59710]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-02-19T21:57:34.174 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:33 smithi196 bash[59710]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-02-19T21:57:34.174 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:33 smithi196 bash[59710]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-19T21:57:34.175 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:33 smithi196 bash[59710]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-19T21:57:34.175 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:33 smithi196 bash[59710]: --> ceph-volume lvm activate successful for osd ID: 4 2024-02-19T21:57:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:32.551145+0000 mgr.smithi131.cjubsp (mgr.14720) 166 : cluster [DBG] pgmap v121: 129 pgs: 30 peering, 9 stale+active+clean, 90 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-02-19T21:57:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:33.042192+0000 mon.smithi131 (mon.0) 380 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-02-19T21:57:35.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T21:57:35.009+0000 7fc5791b2700 1 -- 172.21.15.196:0/1837884345 <== mon.1 v2:172.21.15.196:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557820e6c680 con 0x557820e33400 2024-02-19T21:57:35.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:35 smithi196 bash[59968]: 209ac9ce00bd86c09fcb6ee88a7e99d2be20efb036e3f21d6f3e6641eeac58fb 2024-02-19T21:57:35.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:35 smithi196 systemd[1]: Started Ceph osd.4 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:57:35.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:35.048631+0000 mon.smithi131 (mon.0) 381 : cluster [WRN] Health check failed: Degraded data redundancy: 10/1353 objects degraded (0.739%), 2 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:35.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:35.096751+0000 mon.smithi131 (mon.0) 382 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:35.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:35.097771+0000 mon.smithi131 (mon.0) 383 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:57:35.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:35.048631+0000 mon.smithi131 (mon.0) 381 : cluster [WRN] Health check failed: Degraded data redundancy: 10/1353 objects degraded (0.739%), 2 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:35.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:35.096751+0000 mon.smithi131 (mon.0) 382 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:35.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:35.097771+0000 mon.smithi131 (mon.0) 383 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:57:35.923 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T21:57:35.533+0000 7fc580c40380 -1 Falling back to public interface 2024-02-19T21:57:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:34.551902+0000 mgr.smithi131.cjubsp (mgr.14720) 167 : cluster [DBG] pgmap v122: 129 pgs: 2 active+undersized+degraded, 30 peering, 4 stale+active+clean, 6 active+undersized, 87 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 10/1353 objects degraded (0.739%) 2024-02-19T21:57:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:35.679858+0000 mon.smithi131 (mon.0) 384 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:34.551902+0000 mgr.smithi131.cjubsp (mgr.14720) 167 : cluster [DBG] pgmap v122: 129 pgs: 2 active+undersized+degraded, 30 peering, 4 stale+active+clean, 6 active+undersized, 87 active+clean; 303 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 10/1353 objects degraded (0.739%) 2024-02-19T21:57:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:35.679858+0000 mon.smithi131 (mon.0) 384 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:37.134 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:38.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:36.552853+0000 mgr.smithi131.cjubsp (mgr.14720) 168 : cluster [DBG] pgmap v123: 129 pgs: 30 active+undersized+degraded, 3 peering, 23 active+undersized, 73 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 211/1353 objects degraded (15.595%) 2024-02-19T21:57:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:36.552853+0000 mgr.smithi131.cjubsp (mgr.14720) 168 : cluster [DBG] pgmap v123: 129 pgs: 30 active+undersized+degraded, 3 peering, 23 active+undersized, 73 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 211/1353 objects degraded (15.595%) 2024-02-19T21:57:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:39.107193+0000 mon.smithi131 (mon.0) 385 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-02-19T21:57:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:39.107193+0000 mon.smithi131 (mon.0) 385 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-02-19T21:57:40.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T21:57:40.223+0000 7fc580c40380 -1 osd.4 90 log_to_monitors {default=true} 2024-02-19T21:57:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:38.553523+0000 mgr.smithi131.cjubsp (mgr.14720) 169 : cluster [DBG] pgmap v124: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 220/1353 objects degraded (16.260%) 2024-02-19T21:57:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.287238+0000 mon.smithi131 (mon.0) 386 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.586055+0000 mon.smithi131 (mon.0) 387 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:57:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.586912+0000 mon.smithi131 (mon.0) 388 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:57:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.594073+0000 mon.smithi131 (mon.0) 389 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.600238+0000 mon.smithi131 (mon.0) 390 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.603579+0000 mon.smithi131 (mon.0) 391 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:57:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.605997+0000 mon.smithi131 (mon.0) 392 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.607345+0000 mon.smithi131 (mon.0) 393 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.608647+0000 mon.smithi131 (mon.0) 394 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.610067+0000 mon.smithi131 (mon.0) 395 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-19T21:57:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:38.553523+0000 mgr.smithi131.cjubsp (mgr.14720) 169 : cluster [DBG] pgmap v124: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 220/1353 objects degraded (16.260%) 2024-02-19T21:57:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.287238+0000 mon.smithi131 (mon.0) 386 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.586055+0000 mon.smithi131 (mon.0) 387 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:57:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.586912+0000 mon.smithi131 (mon.0) 388 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:57:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.594073+0000 mon.smithi131 (mon.0) 389 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:40.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.600238+0000 mon.smithi131 (mon.0) 390 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:40.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.603579+0000 mon.smithi131 (mon.0) 391 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:57:40.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.605997+0000 mon.smithi131 (mon.0) 392 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:40.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.607345+0000 mon.smithi131 (mon.0) 393 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:40.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.608647+0000 mon.smithi131 (mon.0) 394 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:40.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.610067+0000 mon.smithi131 (mon.0) 395 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-19T21:57:41.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 21:57:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T21:57:41.186+0000 7fc57392a700 -1 osd.4 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-19T21:57:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:39.610233+0000 mgr.smithi131.cjubsp (mgr.14720) 170 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-19T21:57:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:57:39.610946+0000 mgr.smithi131.cjubsp (mgr.14720) 171 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-02-19T21:57:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:40.227135+0000 mon.smithi196 (mon.1) 2 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/321525468,v1:172.21.15.196:6801/321525468]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-19T21:57:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:40.227667+0000 mon.smithi131 (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-19T21:57:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:39.610233+0000 mgr.smithi131.cjubsp (mgr.14720) 170 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-19T21:57:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:57:39.610946+0000 mgr.smithi131.cjubsp (mgr.14720) 171 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-02-19T21:57:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:40.227135+0000 mon.smithi196 (mon.1) 2 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/321525468,v1:172.21.15.196:6801/321525468]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-19T21:57:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:40.227667+0000 mon.smithi131 (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-19T21:57:42.165 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:42.475 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:40.554311+0000 mgr.smithi131.cjubsp (mgr.14720) 172 : cluster [DBG] pgmap v125: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 215 B/s rd, 107 B/s wr, 0 op/s; 220/1353 objects degraded (16.260%) 2024-02-19T21:57:42.475 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:41.170287+0000 mon.smithi131 (mon.0) 397 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-19T21:57:42.475 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:41.170347+0000 mon.smithi131 (mon.0) 398 : cluster [DBG] osdmap e93: 8 total, 7 up, 8 in 2024-02-19T21:57:42.475 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:41.172497+0000 mon.smithi196 (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/321525468,v1:172.21.15.196:6801/321525468]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:57:42.475 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:41.173192+0000 mon.smithi131 (mon.0) 399 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:57:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:40.554311+0000 mgr.smithi131.cjubsp (mgr.14720) 172 : cluster [DBG] pgmap v125: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 215 B/s rd, 107 B/s wr, 0 op/s; 220/1353 objects degraded (16.260%) 2024-02-19T21:57:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:41.170287+0000 mon.smithi131 (mon.0) 397 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-19T21:57:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:41.170347+0000 mon.smithi131 (mon.0) 398 : cluster [DBG] osdmap e93: 8 total, 7 up, 8 in 2024-02-19T21:57:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:41.172497+0000 mon.smithi196 (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.196:6800/321525468,v1:172.21.15.196:6801/321525468]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:57:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:41.173192+0000 mon.smithi131 (mon.0) 399 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:57:43.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:42.168009+0000 mon.smithi131 (mon.0) 400 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:57:43.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:42.171683+0000 mon.smithi131 (mon.0) 401 : cluster [INF] osd.4 [v2:172.21.15.196:6800/321525468,v1:172.21.15.196:6801/321525468] boot 2024-02-19T21:57:43.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:42.171712+0000 mon.smithi131 (mon.0) 402 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-02-19T21:57:43.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:42.172163+0000 mon.smithi131 (mon.0) 403 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:57:43.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:42.816212+0000 mon.smithi131 (mon.0) 404 : cluster [WRN] Health check update: Degraded data redundancy: 220/1353 objects degraded (16.260%), 32 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:43.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:43.173540+0000 mon.smithi131 (mon.0) 405 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-02-19T21:57:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:42.168009+0000 mon.smithi131 (mon.0) 400 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:57:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:42.171683+0000 mon.smithi131 (mon.0) 401 : cluster [INF] osd.4 [v2:172.21.15.196:6800/321525468,v1:172.21.15.196:6801/321525468] boot 2024-02-19T21:57:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:42.171712+0000 mon.smithi131 (mon.0) 402 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-02-19T21:57:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:42.172163+0000 mon.smithi131 (mon.0) 403 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-19T21:57:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:42.816212+0000 mon.smithi131 (mon.0) 404 : cluster [WRN] Health check update: Degraded data redundancy: 220/1353 objects degraded (16.260%), 32 pgs degraded (PG_DEGRADED) 2024-02-19T21:57:43.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:43.173540+0000 mon.smithi131 (mon.0) 405 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-02-19T21:57:44.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:42.554820+0000 mgr.smithi131.cjubsp (mgr.14720) 173 : cluster [DBG] pgmap v128: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 220/1353 objects degraded (16.260%) 2024-02-19T21:57:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:42.554820+0000 mgr.smithi131.cjubsp (mgr.14720) 173 : cluster [DBG] pgmap v128: 129 pgs: 32 active+undersized+degraded, 24 active+undersized, 73 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 220/1353 objects degraded (16.260%) 2024-02-19T21:57:46.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:44.555471+0000 mgr.smithi131.cjubsp (mgr.14720) 174 : cluster [DBG] pgmap v130: 129 pgs: 30 active+undersized+degraded, 23 active+undersized, 76 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 210/1353 objects degraded (15.521%) 2024-02-19T21:57:46.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:45.811924+0000 mon.smithi131 (mon.0) 406 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:46.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:44.555471+0000 mgr.smithi131.cjubsp (mgr.14720) 174 : cluster [DBG] pgmap v130: 129 pgs: 30 active+undersized+degraded, 23 active+undersized, 76 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 210/1353 objects degraded (15.521%) 2024-02-19T21:57:46.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:45.811924+0000 mon.smithi131 (mon.0) 406 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:47.210 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:48.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:46.556170+0000 mgr.smithi131.cjubsp (mgr.14720) 175 : cluster [DBG] pgmap v131: 129 pgs: 1 active+undersized+degraded, 128 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 1/1353 objects degraded (0.074%) 2024-02-19T21:57:48.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:47.816857+0000 mon.smithi131 (mon.0) 407 : cluster [WRN] Health check update: Degraded data redundancy: 1/1353 objects degraded (0.074%), 1 pg degraded (PG_DEGRADED) 2024-02-19T21:57:48.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:46.556170+0000 mgr.smithi131.cjubsp (mgr.14720) 175 : cluster [DBG] pgmap v131: 129 pgs: 1 active+undersized+degraded, 128 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 1/1353 objects degraded (0.074%) 2024-02-19T21:57:48.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:47.816857+0000 mon.smithi131 (mon.0) 407 : cluster [WRN] Health check update: Degraded data redundancy: 1/1353 objects degraded (0.074%), 1 pg degraded (PG_DEGRADED) 2024-02-19T21:57:49.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:49.201322+0000 mon.smithi131 (mon.0) 408 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/1353 objects degraded (0.074%), 1 pg degraded) 2024-02-19T21:57:49.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:49.201349+0000 mon.smithi131 (mon.0) 409 : cluster [INF] Cluster is now healthy 2024-02-19T21:57:49.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:49.201322+0000 mon.smithi131 (mon.0) 408 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/1353 objects degraded (0.074%), 1 pg degraded) 2024-02-19T21:57:49.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:49.201349+0000 mon.smithi131 (mon.0) 409 : cluster [INF] Cluster is now healthy 2024-02-19T21:57:50.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:48.556804+0000 mgr.smithi131.cjubsp (mgr.14720) 176 : cluster [DBG] pgmap v132: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 277 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:50.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:48.556804+0000 mgr.smithi131.cjubsp (mgr.14720) 176 : cluster [DBG] pgmap v132: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 277 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:50.557362+0000 mgr.smithi131.cjubsp (mgr.14720) 177 : cluster [DBG] pgmap v133: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 244 B/s rd, 122 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:50.680056+0000 mon.smithi131 (mon.0) 410 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:50.557362+0000 mgr.smithi131.cjubsp (mgr.14720) 177 : cluster [DBG] pgmap v133: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 244 B/s rd, 122 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:50.680056+0000 mon.smithi131 (mon.0) 410 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:52.253 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:53.403 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:57:53.972 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:57:53.972 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (10m) 43s ago 12m 21.1M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (2m) 43s ago 12m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (2m) 14s ago 11m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (10m) 43s ago 12m 34.6M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (6m) 43s ago 6m 39.0M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (6m) 14s ago 6m 13.6M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (3m) 43s ago 13m 440M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (3m) 14s ago 11m 388M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (3m) 43s ago 13m 48.0M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (2m) 14s ago 11m 43.8M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (4m) 43s ago 4m 80.0M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (4m) 14s ago 4m 75.7M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (11m) 43s ago 11m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (11m) 14s ago 11m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (2m) 43s ago 10m 113M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T21:57:53.973 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (113s) 43s ago 10m 101M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T21:57:53.974 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (82s) 43s ago 9m 80.6M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T21:57:53.974 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (50s) 43s ago 9m 13.3M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T21:57:53.974 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (19s) 14s ago 9m 10.7M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T21:57:53.974 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (8m) 14s ago 8m 376M 2712M 16.2.5 6933c2a0b7dd f11f526480ad 2024-02-19T21:57:53.974 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (8m) 14s ago 8m 312M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:57:53.974 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (8m) 14s ago 8m 467M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:57:53.974 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (3m) 43s ago 11m 60.5M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:57:54.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:52.558259+0000 mgr.smithi131.cjubsp (mgr.14720) 178 : cluster [DBG] pgmap v134: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:52.558259+0000 mgr.smithi131.cjubsp (mgr.14720) 178 : cluster [DBG] pgmap v134: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:54.311 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:57:54.311 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:57:54.311 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:57:54.311 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:57:54.311 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:57:54.311 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 5, 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 3 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 9, 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 5 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:57:54.312 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:57:54.602 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:57:54.602 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:57:54.602 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:57:54.602 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:57:54.602 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:57:54.602 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T21:57:54.602 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:57:54.603 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:57:54.603 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:57:54.603 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "11/23 daemons upgraded", 2024-02-19T21:57:54.603 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading osd daemons", 2024-02-19T21:57:54.603 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:57:54.603 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:57:55.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:53.391314+0000 mgr.smithi131.cjubsp (mgr.14720) 179 : audit [DBG] from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:55.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:53.682076+0000 mgr.smithi131.cjubsp (mgr.14720) 180 : audit [DBG] from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:55.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:53.964415+0000 mgr.smithi131.cjubsp (mgr.14720) 181 : audit [DBG] from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:55.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:54.312253+0000 mon.smithi131 (mon.0) 411 : audit [DBG] from='client.? 172.21.15.131:0/981768269' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:55.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:54.626197+0000 mon.smithi131 (mon.0) 412 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-19T21:57:55.055 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:53.391314+0000 mgr.smithi131.cjubsp (mgr.14720) 179 : audit [DBG] from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:55.055 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:53.682076+0000 mgr.smithi131.cjubsp (mgr.14720) 180 : audit [DBG] from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:55.055 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:53.964415+0000 mgr.smithi131.cjubsp (mgr.14720) 181 : audit [DBG] from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:55.055 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:54.312253+0000 mon.smithi131 (mon.0) 411 : audit [DBG] from='client.? 172.21.15.131:0/981768269' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:57:55.055 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:54.626197+0000 mon.smithi131 (mon.0) 412 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-19T21:57:56.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:54.558670+0000 mgr.smithi131.cjubsp (mgr.14720) 182 : cluster [DBG] pgmap v135: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 179 B/s rd, 179 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:56.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:54.602801+0000 mgr.smithi131.cjubsp (mgr.14720) 183 : audit [DBG] from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:56.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:54.626341+0000 mgr.smithi131.cjubsp (mgr.14720) 184 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-19T21:57:56.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:57:54.627248+0000 mgr.smithi131.cjubsp (mgr.14720) 185 : cephadm [INF] Upgrade: osd.5 is safe to restart 2024-02-19T21:57:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:54.558670+0000 mgr.smithi131.cjubsp (mgr.14720) 182 : cluster [DBG] pgmap v135: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 179 B/s rd, 179 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:54.602801+0000 mgr.smithi131.cjubsp (mgr.14720) 183 : audit [DBG] from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:57:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:54.626341+0000 mgr.smithi131.cjubsp (mgr.14720) 184 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-19T21:57:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:57:54.627248+0000 mgr.smithi131.cjubsp (mgr.14720) 185 : cephadm [INF] Upgrade: osd.5 is safe to restart 2024-02-19T21:57:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:55.823060+0000 mon.smithi131 (mon.0) 413 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:57:56.261913+0000 mgr.smithi131.cjubsp (mgr.14720) 186 : cephadm [INF] Upgrade: Updating osd.5 2024-02-19T21:57:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:56.266881+0000 mon.smithi131 (mon.0) 414 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:56.267745+0000 mon.smithi131 (mon.0) 415 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-19T21:57:57.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:57:56.268451+0000 mon.smithi131 (mon.0) 416 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:57:57.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:57:56.269807+0000 mgr.smithi131.cjubsp (mgr.14720) 187 : cephadm [INF] Deploying daemon osd.5 on smithi196 2024-02-19T21:57:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:55.823060+0000 mon.smithi131 (mon.0) 413 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:57:56.261913+0000 mgr.smithi131.cjubsp (mgr.14720) 186 : cephadm [INF] Upgrade: Updating osd.5 2024-02-19T21:57:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:56.266881+0000 mon.smithi131 (mon.0) 414 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:57:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:56.267745+0000 mon.smithi131 (mon.0) 415 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-19T21:57:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:57:56.268451+0000 mon.smithi131 (mon.0) 416 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:57:57.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:57:56.269807+0000 mgr.smithi131.cjubsp (mgr.14720) 187 : cephadm [INF] Deploying daemon osd.5 on smithi196 2024-02-19T21:57:57.287 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:57:58.659 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:57:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:56.559258+0000 mgr.smithi131.cjubsp (mgr.14720) 188 : cluster [DBG] pgmap v136: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:58.663 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:57:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:56.559258+0000 mgr.smithi131.cjubsp (mgr.14720) 188 : cluster [DBG] pgmap v136: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:57:59.332 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:57:59 smithi196 systemd[1]: Stopping Ceph osd.5 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:57:59.673 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:57:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.5[43428]: debug 2024-02-19T21:57:59.331+0000 7f77f7124700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T21:57:59.673 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:57:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.5[43428]: debug 2024-02-19T21:57:59.331+0000 7f77f7124700 -1 osd.5 95 *** Got signal Terminated *** 2024-02-19T21:57:59.673 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:57:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.5[43428]: debug 2024-02-19T21:57:59.331+0000 7f77f7124700 -1 osd.5 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T21:58:00.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:57:59 smithi196 bash[61370]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.5 2024-02-19T21:58:00.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:58.559825+0000 mgr.smithi131.cjubsp (mgr.14720) 189 : cluster [DBG] pgmap v137: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:58:00.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.615296+0000 mon.smithi131 (mon.0) 417 : cluster [DBG] osd.5 reported immediately failed by osd.2 2024-02-19T21:58:00.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.615350+0000 mon.smithi131 (mon.0) 418 : cluster [INF] osd.5 failed (root=default,host=smithi196) (connection refused reported by osd.2) 2024-02-19T21:58:00.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.615518+0000 mon.smithi131 (mon.0) 419 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.615613+0000 mon.smithi131 (mon.0) 420 : cluster [DBG] osd.5 reported immediately failed by osd.2 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.615704+0000 mon.smithi131 (mon.0) 421 : cluster [DBG] osd.5 reported immediately failed by osd.3 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.615853+0000 mon.smithi131 (mon.0) 422 : cluster [DBG] osd.5 reported immediately failed by osd.4 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.615959+0000 mon.smithi131 (mon.0) 423 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616056+0000 mon.smithi131 (mon.0) 424 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616147+0000 mon.smithi131 (mon.0) 425 : cluster [DBG] osd.5 reported immediately failed by osd.0 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616227+0000 mon.smithi131 (mon.0) 426 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616309+0000 mon.smithi131 (mon.0) 427 : cluster [DBG] osd.5 reported immediately failed by osd.3 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616430+0000 mon.smithi131 (mon.0) 428 : cluster [DBG] osd.5 reported immediately failed by osd.4 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616521+0000 mon.smithi131 (mon.0) 429 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616601+0000 mon.smithi131 (mon.0) 430 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616687+0000 mon.smithi131 (mon.0) 431 : cluster [DBG] osd.5 reported immediately failed by osd.0 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616771+0000 mon.smithi131 (mon.0) 432 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.616872+0000 mon.smithi131 (mon.0) 433 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:57:59.852929+0000 mon.smithi131 (mon.0) 434 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016066+0000 mon.smithi131 (mon.0) 435 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016170+0000 mon.smithi131 (mon.0) 436 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016348+0000 mon.smithi131 (mon.0) 437 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016453+0000 mon.smithi131 (mon.0) 438 : cluster [DBG] osd.5 reported immediately failed by osd.2 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016550+0000 mon.smithi131 (mon.0) 439 : cluster [DBG] osd.5 reported immediately failed by osd.3 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016679+0000 mon.smithi131 (mon.0) 440 : cluster [DBG] osd.5 reported immediately failed by osd.4 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016765+0000 mon.smithi131 (mon.0) 441 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016877+0000 mon.smithi131 (mon.0) 442 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.016965+0000 mon.smithi131 (mon.0) 443 : cluster [DBG] osd.5 reported immediately failed by osd.0 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.017048+0000 mon.smithi131 (mon.0) 444 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.017133+0000 mon.smithi131 (mon.0) 445 : cluster [DBG] osd.5 reported immediately failed by osd.2 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.017219+0000 mon.smithi131 (mon.0) 446 : cluster [DBG] osd.5 reported immediately failed by osd.3 2024-02-19T21:58:00.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.017339+0000 mon.smithi131 (mon.0) 447 : cluster [DBG] osd.5 reported immediately failed by osd.4 2024-02-19T21:58:00.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.017421+0000 mon.smithi131 (mon.0) 448 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.017471+0000 mon.smithi131 (mon.0) 449 : cluster [DBG] osd.5 reported immediately failed by osd.0 2024-02-19T21:58:00.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.017587+0000 mon.smithi131 (mon.0) 450 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.676 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.053519+0000 mon.smithi131 (mon.0) 451 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:58.559825+0000 mgr.smithi131.cjubsp (mgr.14720) 189 : cluster [DBG] pgmap v137: 129 pgs: 129 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-02-19T21:58:00.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.615296+0000 mon.smithi131 (mon.0) 417 : cluster [DBG] osd.5 reported immediately failed by osd.2 2024-02-19T21:58:00.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.615350+0000 mon.smithi131 (mon.0) 418 : cluster [INF] osd.5 failed (root=default,host=smithi196) (connection refused reported by osd.2) 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.615518+0000 mon.smithi131 (mon.0) 419 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.615613+0000 mon.smithi131 (mon.0) 420 : cluster [DBG] osd.5 reported immediately failed by osd.2 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.615704+0000 mon.smithi131 (mon.0) 421 : cluster [DBG] osd.5 reported immediately failed by osd.3 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.615853+0000 mon.smithi131 (mon.0) 422 : cluster [DBG] osd.5 reported immediately failed by osd.4 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.615959+0000 mon.smithi131 (mon.0) 423 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616056+0000 mon.smithi131 (mon.0) 424 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616147+0000 mon.smithi131 (mon.0) 425 : cluster [DBG] osd.5 reported immediately failed by osd.0 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616227+0000 mon.smithi131 (mon.0) 426 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616309+0000 mon.smithi131 (mon.0) 427 : cluster [DBG] osd.5 reported immediately failed by osd.3 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616430+0000 mon.smithi131 (mon.0) 428 : cluster [DBG] osd.5 reported immediately failed by osd.4 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616521+0000 mon.smithi131 (mon.0) 429 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616601+0000 mon.smithi131 (mon.0) 430 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616687+0000 mon.smithi131 (mon.0) 431 : cluster [DBG] osd.5 reported immediately failed by osd.0 2024-02-19T21:58:00.757 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616771+0000 mon.smithi131 (mon.0) 432 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.616872+0000 mon.smithi131 (mon.0) 433 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:57:59.852929+0000 mon.smithi131 (mon.0) 434 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016066+0000 mon.smithi131 (mon.0) 435 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016170+0000 mon.smithi131 (mon.0) 436 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016348+0000 mon.smithi131 (mon.0) 437 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016453+0000 mon.smithi131 (mon.0) 438 : cluster [DBG] osd.5 reported immediately failed by osd.2 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016550+0000 mon.smithi131 (mon.0) 439 : cluster [DBG] osd.5 reported immediately failed by osd.3 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016679+0000 mon.smithi131 (mon.0) 440 : cluster [DBG] osd.5 reported immediately failed by osd.4 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016765+0000 mon.smithi131 (mon.0) 441 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016877+0000 mon.smithi131 (mon.0) 442 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.016965+0000 mon.smithi131 (mon.0) 443 : cluster [DBG] osd.5 reported immediately failed by osd.0 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.017048+0000 mon.smithi131 (mon.0) 444 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.017133+0000 mon.smithi131 (mon.0) 445 : cluster [DBG] osd.5 reported immediately failed by osd.2 2024-02-19T21:58:00.758 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.017219+0000 mon.smithi131 (mon.0) 446 : cluster [DBG] osd.5 reported immediately failed by osd.3 2024-02-19T21:58:00.759 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.017339+0000 mon.smithi131 (mon.0) 447 : cluster [DBG] osd.5 reported immediately failed by osd.4 2024-02-19T21:58:00.759 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.017421+0000 mon.smithi131 (mon.0) 448 : cluster [DBG] osd.5 reported immediately failed by osd.7 2024-02-19T21:58:00.759 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.017471+0000 mon.smithi131 (mon.0) 449 : cluster [DBG] osd.5 reported immediately failed by osd.0 2024-02-19T21:58:00.759 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.017587+0000 mon.smithi131 (mon.0) 450 : cluster [DBG] osd.5 reported immediately failed by osd.1 2024-02-19T21:58:00.759 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.053519+0000 mon.smithi131 (mon.0) 451 : cluster [DBG] osd.5 reported immediately failed by osd.6 2024-02-19T21:58:01.549 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.268996+0000 mon.smithi131 (mon.0) 452 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:58:01.549 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.277056+0000 mon.smithi131 (mon.0) 453 : cluster [DBG] osdmap e96: 8 total, 7 up, 8 in 2024-02-19T21:58:01.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.268996+0000 mon.smithi131 (mon.0) 452 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:58:01.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.277056+0000 mon.smithi131 (mon.0) 453 : cluster [DBG] osdmap e96: 8 total, 7 up, 8 in 2024-02-19T21:58:02.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:01 smithi196 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.5.service: Succeeded. 2024-02-19T21:58:02.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:01 smithi196 systemd[1]: Stopped Ceph osd.5 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:58:02.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:01 smithi196 systemd[1]: Starting Ceph osd.5 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:58:02.324 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:02.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:00.560395+0000 mgr.smithi131.cjubsp (mgr.14720) 190 : cluster [DBG] pgmap v139: 129 pgs: 18 stale+active+clean, 111 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:58:02.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:01.285220+0000 mon.smithi131 (mon.0) 454 : cluster [DBG] osdmap e97: 8 total, 7 up, 8 in 2024-02-19T21:58:02.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:00.560395+0000 mgr.smithi131.cjubsp (mgr.14720) 190 : cluster [DBG] pgmap v139: 129 pgs: 18 stale+active+clean, 111 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:58:02.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:01.285220+0000 mon.smithi131 (mon.0) 454 : cluster [DBG] osdmap e97: 8 total, 7 up, 8 in 2024-02-19T21:58:03.174 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:02 smithi196 bash[61677]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-02-19T21:58:03.174 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:02 smithi196 bash[61677]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-19T21:58:03.174 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:02 smithi196 bash[61677]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-02-19T21:58:03.174 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:02 smithi196 bash[61677]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-02-19T21:58:03.174 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:02 smithi196 bash[61677]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-02-19T21:58:03.175 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:02 smithi196 bash[61677]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-19T21:58:03.175 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:02 smithi196 bash[61677]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-19T21:58:03.175 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:02 smithi196 bash[61677]: --> ceph-volume lvm activate successful for osd ID: 5 2024-02-19T21:58:03.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:02.561109+0000 mgr.smithi131.cjubsp (mgr.14720) 191 : cluster [DBG] pgmap v141: 129 pgs: 9 active+undersized+degraded, 15 peering, 5 stale+active+clean, 10 active+undersized, 90 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 56/1353 objects degraded (4.139%) 2024-02-19T21:58:03.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:02.561109+0000 mgr.smithi131.cjubsp (mgr.14720) 191 : cluster [DBG] pgmap v141: 129 pgs: 9 active+undersized+degraded, 15 peering, 5 stale+active+clean, 10 active+undersized, 90 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 56/1353 objects degraded (4.139%) 2024-02-19T21:58:04.279 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T21:58:04.127+0000 7fe43a459700 1 -- 172.21.15.196:0/2750993320 <== mon.1 v2:172.21.15.196:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56333f72c680 con 0x56333f6f3400 2024-02-19T21:58:04.279 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:04 smithi196 bash[61918]: 4c4b41297954af92538450aa8fd6c5a868f296596d3b96a9852bbc044ea912f0 2024-02-19T21:58:04.279 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:04 smithi196 systemd[1]: Started Ceph osd.5 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:58:04.643 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:03.273266+0000 mon.smithi131 (mon.0) 455 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-02-19T21:58:04.644 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:03.273329+0000 mon.smithi131 (mon.0) 456 : cluster [WRN] Health check failed: Degraded data redundancy: 56/1353 objects degraded (4.139%), 9 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:04.644 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:04.240157+0000 mon.smithi131 (mon.0) 457 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:04.644 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:04.241244+0000 mon.smithi131 (mon.0) 458 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:58:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:03.273266+0000 mon.smithi131 (mon.0) 455 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-02-19T21:58:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:03.273329+0000 mon.smithi131 (mon.0) 456 : cluster [WRN] Health check failed: Degraded data redundancy: 56/1353 objects degraded (4.139%), 9 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:04.240157+0000 mon.smithi131 (mon.0) 457 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:04.241244+0000 mon.smithi131 (mon.0) 458 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:58:04.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T21:58:04.643+0000 7fe441ee7380 -1 Falling back to public interface 2024-02-19T21:58:05.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:04.561593+0000 mgr.smithi131.cjubsp (mgr.14720) 192 : cluster [DBG] pgmap v142: 129 pgs: 14 active+undersized+degraded, 15 peering, 1 stale+active+clean, 11 active+undersized, 88 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 96/1353 objects degraded (7.095%) 2024-02-19T21:58:05.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:04.561593+0000 mgr.smithi131.cjubsp (mgr.14720) 192 : cluster [DBG] pgmap v142: 129 pgs: 14 active+undersized+degraded, 15 peering, 1 stale+active+clean, 11 active+undersized, 88 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s; 96/1353 objects degraded (7.095%) 2024-02-19T21:58:06.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:05.680629+0000 mon.smithi131 (mon.0) 459 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:07.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:05.680629+0000 mon.smithi131 (mon.0) 459 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:07.361 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:08.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:06.562454+0000 mgr.smithi131.cjubsp (mgr.14720) 193 : cluster [DBG] pgmap v143: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 155/1353 objects degraded (11.456%) 2024-02-19T21:58:08.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:06.678438+0000 mon.smithi131 (mon.0) 460 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 7 pgs peering) 2024-02-19T21:58:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:06.562454+0000 mgr.smithi131.cjubsp (mgr.14720) 193 : cluster [DBG] pgmap v143: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 155/1353 objects degraded (11.456%) 2024-02-19T21:58:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:06.678438+0000 mon.smithi131 (mon.0) 460 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 7 pgs peering) 2024-02-19T21:58:09.423 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T21:58:09.339+0000 7fe441ee7380 -1 osd.5 95 log_to_monitors {default=true} 2024-02-19T21:58:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.123621+0000 mon.smithi131 (mon.0) 461 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.124554+0000 mon.smithi131 (mon.0) 462 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:58:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.125353+0000 mon.smithi131 (mon.0) 463 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:58:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.130022+0000 mon.smithi131 (mon.0) 464 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.136101+0000 mon.smithi131 (mon.0) 465 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.139443+0000 mon.smithi131 (mon.0) 466 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:58:09.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.141736+0000 mon.smithi131 (mon.0) 467 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:09.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.142986+0000 mon.smithi131 (mon.0) 468 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:09.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.144271+0000 mon.smithi131 (mon.0) 469 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:09.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.145752+0000 mon.smithi131 (mon.0) 470 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-19T21:58:09.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:08.145929+0000 mgr.smithi131.cjubsp (mgr.14720) 194 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-19T21:58:09.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:58:08.146742+0000 mgr.smithi131.cjubsp (mgr.14720) 195 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-02-19T21:58:09.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.123621+0000 mon.smithi131 (mon.0) 461 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:09.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.124554+0000 mon.smithi131 (mon.0) 462 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:58:09.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.125353+0000 mon.smithi131 (mon.0) 463 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:58:09.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.130022+0000 mon.smithi131 (mon.0) 464 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:09.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.136101+0000 mon.smithi131 (mon.0) 465 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:09.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.139443+0000 mon.smithi131 (mon.0) 466 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:58:09.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.141736+0000 mon.smithi131 (mon.0) 467 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:09.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.142986+0000 mon.smithi131 (mon.0) 468 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:09.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.144271+0000 mon.smithi131 (mon.0) 469 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:09.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.145752+0000 mon.smithi131 (mon.0) 470 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-19T21:58:09.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:08.145929+0000 mgr.smithi131.cjubsp (mgr.14720) 194 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-19T21:58:09.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:58:08.146742+0000 mgr.smithi131.cjubsp (mgr.14720) 195 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-02-19T21:58:10.423 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 21:58:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T21:58:10.161+0000 7fe4333ce700 -1 osd.5 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-19T21:58:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:08.563026+0000 mgr.smithi131.cjubsp (mgr.14720) 196 : cluster [DBG] pgmap v144: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 247 B/s wr, 0 op/s; 155/1353 objects degraded (11.456%) 2024-02-19T21:58:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:09.343010+0000 mon.smithi196 (mon.1) 4 : audit [INF] from='osd.5 [v2:172.21.15.196:6808/2090776581,v1:172.21.15.196:6809/2090776581]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-19T21:58:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:09.343470+0000 mon.smithi131 (mon.0) 471 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-19T21:58:10.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:08.563026+0000 mgr.smithi131.cjubsp (mgr.14720) 196 : cluster [DBG] pgmap v144: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 303 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 247 B/s wr, 0 op/s; 155/1353 objects degraded (11.456%) 2024-02-19T21:58:10.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:09.343010+0000 mon.smithi196 (mon.1) 4 : audit [INF] from='osd.5 [v2:172.21.15.196:6808/2090776581,v1:172.21.15.196:6809/2090776581]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-19T21:58:10.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:09.343470+0000 mon.smithi131 (mon.0) 471 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-19T21:58:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:10.140979+0000 mon.smithi131 (mon.0) 472 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-19T21:58:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:10.141067+0000 mon.smithi131 (mon.0) 473 : cluster [DBG] osdmap e98: 8 total, 7 up, 8 in 2024-02-19T21:58:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:10.144023+0000 mon.smithi196 (mon.1) 5 : audit [INF] from='osd.5 [v2:172.21.15.196:6808/2090776581,v1:172.21.15.196:6809/2090776581]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:58:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:10.149011+0000 mon.smithi131 (mon.0) 474 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:58:11.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:10.140979+0000 mon.smithi131 (mon.0) 472 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-19T21:58:11.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:10.141067+0000 mon.smithi131 (mon.0) 473 : cluster [DBG] osdmap e98: 8 total, 7 up, 8 in 2024-02-19T21:58:11.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:10.144023+0000 mon.smithi196 (mon.1) 5 : audit [INF] from='osd.5 [v2:172.21.15.196:6808/2090776581,v1:172.21.15.196:6809/2090776581]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:58:11.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:10.149011+0000 mon.smithi131 (mon.0) 474 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:58:12.399 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:10.563661+0000 mgr.smithi131.cjubsp (mgr.14720) 197 : cluster [DBG] pgmap v146: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 220 B/s rd, 220 B/s wr, 0 op/s; 155/1353 objects degraded (11.456%) 2024-02-19T21:58:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:11.144767+0000 mon.smithi131 (mon.0) 475 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:58:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:11.149890+0000 mon.smithi131 (mon.0) 476 : cluster [INF] osd.5 [v2:172.21.15.196:6808/2090776581,v1:172.21.15.196:6809/2090776581] boot 2024-02-19T21:58:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:11.149957+0000 mon.smithi131 (mon.0) 477 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-02-19T21:58:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:11.152249+0000 mon.smithi131 (mon.0) 478 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:58:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:12.150740+0000 mon.smithi131 (mon.0) 479 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-02-19T21:58:12.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:10.563661+0000 mgr.smithi131.cjubsp (mgr.14720) 197 : cluster [DBG] pgmap v146: 129 pgs: 24 active+undersized+degraded, 21 active+undersized, 84 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 220 B/s rd, 220 B/s wr, 0 op/s; 155/1353 objects degraded (11.456%) 2024-02-19T21:58:12.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:11.144767+0000 mon.smithi131 (mon.0) 475 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:58:12.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:11.149890+0000 mon.smithi131 (mon.0) 476 : cluster [INF] osd.5 [v2:172.21.15.196:6808/2090776581,v1:172.21.15.196:6809/2090776581] boot 2024-02-19T21:58:12.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:11.149957+0000 mon.smithi131 (mon.0) 477 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-02-19T21:58:12.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:11.152249+0000 mon.smithi131 (mon.0) 478 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-19T21:58:12.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:12.150740+0000 mon.smithi131 (mon.0) 479 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-02-19T21:58:13.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:12.821516+0000 mon.smithi131 (mon.0) 480 : cluster [WRN] Health check update: Degraded data redundancy: 155/1353 objects degraded (11.456%), 24 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:13.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:12.821516+0000 mon.smithi131 (mon.0) 480 : cluster [WRN] Health check update: Degraded data redundancy: 155/1353 objects degraded (11.456%), 24 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:14.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:12.564232+0000 mgr.smithi131.cjubsp (mgr.14720) 198 : cluster [DBG] pgmap v149: 129 pgs: 15 peering, 17 active+undersized+degraded, 13 active+undersized, 84 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 341 B/s wr, 0 op/s; 116/1353 objects degraded (8.574%) 2024-02-19T21:58:14.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:12.564232+0000 mgr.smithi131.cjubsp (mgr.14720) 198 : cluster [DBG] pgmap v149: 129 pgs: 15 peering, 17 active+undersized+degraded, 13 active+undersized, 84 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 341 B/s wr, 0 op/s; 116/1353 objects degraded (8.574%) 2024-02-19T21:58:16.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:14.564994+0000 mgr.smithi131.cjubsp (mgr.14720) 199 : cluster [DBG] pgmap v150: 129 pgs: 15 peering, 15 active+undersized+degraded, 13 active+undersized, 86 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 102/1353 objects degraded (7.539%) 2024-02-19T21:58:16.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:15.839329+0000 mon.smithi131 (mon.0) 481 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:16.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:14.564994+0000 mgr.smithi131.cjubsp (mgr.14720) 199 : cluster [DBG] pgmap v150: 129 pgs: 15 peering, 15 active+undersized+degraded, 13 active+undersized, 86 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 102/1353 objects degraded (7.539%) 2024-02-19T21:58:16.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:15.839329+0000 mon.smithi131 (mon.0) 481 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:17.447 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:17.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:17.174107+0000 mon.smithi131 (mon.0) 482 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 102/1353 objects degraded (7.539%), 15 pgs degraded) 2024-02-19T21:58:17.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:17.174137+0000 mon.smithi131 (mon.0) 483 : cluster [INF] Cluster is now healthy 2024-02-19T21:58:17.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:17.174107+0000 mon.smithi131 (mon.0) 482 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 102/1353 objects degraded (7.539%), 15 pgs degraded) 2024-02-19T21:58:17.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:17.174137+0000 mon.smithi131 (mon.0) 483 : cluster [INF] Cluster is now healthy 2024-02-19T21:58:18.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:16.565631+0000 mgr.smithi131.cjubsp (mgr.14720) 200 : cluster [DBG] pgmap v151: 129 pgs: 4 peering, 125 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 159 B/s wr, 0 op/s 2024-02-19T21:58:18.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:16.565631+0000 mgr.smithi131.cjubsp (mgr.14720) 200 : cluster [DBG] pgmap v151: 129 pgs: 4 peering, 125 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 159 B/s wr, 0 op/s 2024-02-19T21:58:20.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:18.566150+0000 mgr.smithi131.cjubsp (mgr.14720) 201 : cluster [DBG] pgmap v152: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-02-19T21:58:20.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:18.566150+0000 mgr.smithi131.cjubsp (mgr.14720) 201 : cluster [DBG] pgmap v152: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-02-19T21:58:22.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:20.566653+0000 mgr.smithi131.cjubsp (mgr.14720) 202 : cluster [DBG] pgmap v153: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 217 B/s rd, 217 B/s wr, 0 op/s 2024-02-19T21:58:22.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:20.682831+0000 mon.smithi131 (mon.0) 484 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:20.566653+0000 mgr.smithi131.cjubsp (mgr.14720) 202 : cluster [DBG] pgmap v153: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 217 B/s rd, 217 B/s wr, 0 op/s 2024-02-19T21:58:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:20.682831+0000 mon.smithi131 (mon.0) 484 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:22.486 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:23.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:22.567331+0000 mgr.smithi131.cjubsp (mgr.14720) 203 : cluster [DBG] pgmap v154: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 196 B/s rd, 196 B/s wr, 0 op/s 2024-02-19T21:58:23.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:23.162399+0000 mon.smithi131 (mon.0) 485 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-19T21:58:23.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:23.162628+0000 mgr.smithi131.cjubsp (mgr.14720) 204 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-19T21:58:23.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:58:23.163934+0000 mgr.smithi131.cjubsp (mgr.14720) 205 : cephadm [INF] Upgrade: osd.6 is safe to restart 2024-02-19T21:58:24.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:22.567331+0000 mgr.smithi131.cjubsp (mgr.14720) 203 : cluster [DBG] pgmap v154: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 196 B/s rd, 196 B/s wr, 0 op/s 2024-02-19T21:58:24.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:23.162399+0000 mon.smithi131 (mon.0) 485 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-19T21:58:24.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:23.162628+0000 mgr.smithi131.cjubsp (mgr.14720) 204 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-19T21:58:24.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:58:23.163934+0000 mgr.smithi131.cjubsp (mgr.14720) 205 : cephadm [INF] Upgrade: osd.6 is safe to restart 2024-02-19T21:58:24.923 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:58:25.493 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (11m) 75s ago 12m 21.1M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (3m) 75s ago 12m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (3m) 17s ago 11m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (11m) 75s ago 12m 34.6M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (7m) 75s ago 7m 39.0M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (7m) 17s ago 7m 13.7M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (4m) 75s ago 14m 440M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (3m) 17s ago 11m 389M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (3m) 75s ago 14m 48.0M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (3m) 17s ago 11m 47.2M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (4m) 75s ago 4m 80.0M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (4m) 17s ago 4m 77.5M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (12m) 75s ago 12m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:58:25.494 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (11m) 17s ago 11m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (2m) 75s ago 11m 113M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (2m) 75s ago 10m 101M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (114s) 75s ago 10m 80.6M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (82s) 75s ago 10m 13.3M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (50s) 17s ago 9m 117M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (21s) 17s ago 9m 11.5M 2712M 16.2.14-548-g81bd20d6 823b921721f5 4c4b41297954 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (9m) 17s ago 9m 312M 2712M 16.2.5 6933c2a0b7dd 51a82b04b836 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (8m) 17s ago 8m 466M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:58:25.495 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (4m) 75s ago 12m 60.5M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:58:25.833 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 6, 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 10, 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 4 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:58:25.834 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:58:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:24.567866+0000 mgr.smithi131.cjubsp (mgr.14720) 206 : cluster [DBG] pgmap v155: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:58:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:58:24.688259+0000 mgr.smithi131.cjubsp (mgr.14720) 207 : cephadm [INF] Upgrade: Updating osd.6 2024-02-19T21:58:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:24.694448+0000 mon.smithi131 (mon.0) 486 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:24.695229+0000 mon.smithi131 (mon.0) 487 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-19T21:58:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:24.695817+0000 mon.smithi131 (mon.0) 488 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:58:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:58:24.697003+0000 mgr.smithi131.cjubsp (mgr.14720) 208 : cephadm [INF] Deploying daemon osd.6 on smithi196 2024-02-19T21:58:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:24.911937+0000 mgr.smithi131.cjubsp (mgr.14720) 209 : audit [DBG] from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:26.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:25.200266+0000 mgr.smithi131.cjubsp (mgr.14720) 210 : audit [DBG] from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:58:26.123 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "12/23 daemons upgraded", 2024-02-19T21:58:26.124 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading osd daemons", 2024-02-19T21:58:26.124 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:58:26.124 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:58:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:24.567866+0000 mgr.smithi131.cjubsp (mgr.14720) 206 : cluster [DBG] pgmap v155: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:58:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:58:24.688259+0000 mgr.smithi131.cjubsp (mgr.14720) 207 : cephadm [INF] Upgrade: Updating osd.6 2024-02-19T21:58:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:24.694448+0000 mon.smithi131 (mon.0) 486 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:24.695229+0000 mon.smithi131 (mon.0) 487 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-19T21:58:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:24.695817+0000 mon.smithi131 (mon.0) 488 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:58:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:58:24.697003+0000 mgr.smithi131.cjubsp (mgr.14720) 208 : cephadm [INF] Deploying daemon osd.6 on smithi196 2024-02-19T21:58:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:24.911937+0000 mgr.smithi131.cjubsp (mgr.14720) 209 : audit [DBG] from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:26.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:25.200266+0000 mgr.smithi131.cjubsp (mgr.14720) 210 : audit [DBG] from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:25.485395+0000 mgr.smithi131.cjubsp (mgr.14720) 211 : audit [DBG] from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:25.833878+0000 mon.smithi131 (mon.0) 489 : audit [DBG] from='client.? 172.21.15.131:0/2960625296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:25.850667+0000 mon.smithi131 (mon.0) 490 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:26.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:26.123785+0000 mgr.smithi131.cjubsp (mgr.14720) 212 : audit [DBG] from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:27.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:25.485395+0000 mgr.smithi131.cjubsp (mgr.14720) 211 : audit [DBG] from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:27.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:25.833878+0000 mon.smithi131 (mon.0) 489 : audit [DBG] from='client.? 172.21.15.131:0/2960625296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:27.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:25.850667+0000 mon.smithi131 (mon.0) 490 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:27.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:26.123785+0000 mgr.smithi131.cjubsp (mgr.14720) 212 : audit [DBG] from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:27.536 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:28.027 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:26.568567+0000 mgr.smithi131.cjubsp (mgr.14720) 213 : cluster [DBG] pgmap v156: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:58:28.028 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:27 smithi196 systemd[1]: Stopping Ceph osd.6 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:58:28.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:26.568567+0000 mgr.smithi131.cjubsp (mgr.14720) 213 : cluster [DBG] pgmap v156: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:58:28.396 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6[46302]: debug 2024-02-19T21:58:28.026+0000 7efc0975a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T21:58:28.396 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6[46302]: debug 2024-02-19T21:58:28.026+0000 7efc0975a700 -1 osd.6 100 *** Got signal Terminated *** 2024-02-19T21:58:28.396 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6[46302]: debug 2024-02-19T21:58:28.026+0000 7efc0975a700 -1 osd.6 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T21:58:28.675 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:28 smithi196 bash[63258]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6 2024-02-19T21:58:29.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.165931+0000 mon.smithi131 (mon.0) 491 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.165966+0000 mon.smithi131 (mon.0) 492 : cluster [INF] osd.6 failed (root=default,host=smithi196) (connection refused reported by osd.4) 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.209624+0000 mon.smithi131 (mon.0) 493 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.282417+0000 mon.smithi131 (mon.0) 494 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.286194+0000 mon.smithi131 (mon.0) 495 : cluster [DBG] osd.6 reported immediately failed by osd.1 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.286394+0000 mon.smithi131 (mon.0) 496 : cluster [DBG] osd.6 reported immediately failed by osd.2 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.286556+0000 mon.smithi131 (mon.0) 497 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.286659+0000 mon.smithi131 (mon.0) 498 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.286765+0000 mon.smithi131 (mon.0) 499 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.286876+0000 mon.smithi131 (mon.0) 500 : cluster [DBG] osd.6 reported immediately failed by osd.1 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.286972+0000 mon.smithi131 (mon.0) 501 : cluster [DBG] osd.6 reported immediately failed by osd.2 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.287028+0000 mon.smithi131 (mon.0) 502 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.287251+0000 mon.smithi131 (mon.0) 503 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.287425+0000 mon.smithi131 (mon.0) 504 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.287544+0000 mon.smithi131 (mon.0) 505 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.287630+0000 mon.smithi131 (mon.0) 506 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.287779+0000 mon.smithi131 (mon.0) 507 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.287925+0000 mon.smithi131 (mon.0) 508 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.288008+0000 mon.smithi131 (mon.0) 509 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.309848+0000 mon.smithi131 (mon.0) 510 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.366606+0000 mon.smithi131 (mon.0) 511 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.410864+0000 mon.smithi131 (mon.0) 512 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687193+0000 mon.smithi131 (mon.0) 513 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687266+0000 mon.smithi131 (mon.0) 514 : cluster [DBG] osd.6 reported immediately failed by osd.1 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687400+0000 mon.smithi131 (mon.0) 515 : cluster [DBG] osd.6 reported immediately failed by osd.2 2024-02-19T21:58:29.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687449+0000 mon.smithi131 (mon.0) 516 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687588+0000 mon.smithi131 (mon.0) 517 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687637+0000 mon.smithi131 (mon.0) 518 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687762+0000 mon.smithi131 (mon.0) 519 : cluster [DBG] osd.6 reported immediately failed by osd.1 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687829+0000 mon.smithi131 (mon.0) 520 : cluster [DBG] osd.6 reported immediately failed by osd.2 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.687965+0000 mon.smithi131 (mon.0) 521 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.688134+0000 mon.smithi131 (mon.0) 522 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.688221+0000 mon.smithi131 (mon.0) 523 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.688305+0000 mon.smithi131 (mon.0) 524 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.688457+0000 mon.smithi131 (mon.0) 525 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.688562+0000 mon.smithi131 (mon.0) 526 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.767905+0000 mon.smithi131 (mon.0) 527 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.811836+0000 mon.smithi131 (mon.0) 528 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.173 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:28 smithi196 bash[63313]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6" found: no such container 2024-02-19T21:58:29.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.165931+0000 mon.smithi131 (mon.0) 491 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.165966+0000 mon.smithi131 (mon.0) 492 : cluster [INF] osd.6 failed (root=default,host=smithi196) (connection refused reported by osd.4) 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.209624+0000 mon.smithi131 (mon.0) 493 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.282417+0000 mon.smithi131 (mon.0) 494 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.286194+0000 mon.smithi131 (mon.0) 495 : cluster [DBG] osd.6 reported immediately failed by osd.1 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.286394+0000 mon.smithi131 (mon.0) 496 : cluster [DBG] osd.6 reported immediately failed by osd.2 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.286556+0000 mon.smithi131 (mon.0) 497 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.286659+0000 mon.smithi131 (mon.0) 498 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.286765+0000 mon.smithi131 (mon.0) 499 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.286876+0000 mon.smithi131 (mon.0) 500 : cluster [DBG] osd.6 reported immediately failed by osd.1 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.286972+0000 mon.smithi131 (mon.0) 501 : cluster [DBG] osd.6 reported immediately failed by osd.2 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.287028+0000 mon.smithi131 (mon.0) 502 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.287251+0000 mon.smithi131 (mon.0) 503 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.287425+0000 mon.smithi131 (mon.0) 504 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.287544+0000 mon.smithi131 (mon.0) 505 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.287630+0000 mon.smithi131 (mon.0) 506 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.287779+0000 mon.smithi131 (mon.0) 507 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.287925+0000 mon.smithi131 (mon.0) 508 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.288008+0000 mon.smithi131 (mon.0) 509 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.309848+0000 mon.smithi131 (mon.0) 510 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.366606+0000 mon.smithi131 (mon.0) 511 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.410864+0000 mon.smithi131 (mon.0) 512 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687193+0000 mon.smithi131 (mon.0) 513 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687266+0000 mon.smithi131 (mon.0) 514 : cluster [DBG] osd.6 reported immediately failed by osd.1 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687400+0000 mon.smithi131 (mon.0) 515 : cluster [DBG] osd.6 reported immediately failed by osd.2 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687449+0000 mon.smithi131 (mon.0) 516 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687588+0000 mon.smithi131 (mon.0) 517 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687637+0000 mon.smithi131 (mon.0) 518 : cluster [DBG] osd.6 reported immediately failed by osd.0 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687762+0000 mon.smithi131 (mon.0) 519 : cluster [DBG] osd.6 reported immediately failed by osd.1 2024-02-19T21:58:29.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687829+0000 mon.smithi131 (mon.0) 520 : cluster [DBG] osd.6 reported immediately failed by osd.2 2024-02-19T21:58:29.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.687965+0000 mon.smithi131 (mon.0) 521 : cluster [DBG] osd.6 reported immediately failed by osd.3 2024-02-19T21:58:29.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.688134+0000 mon.smithi131 (mon.0) 522 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.688221+0000 mon.smithi131 (mon.0) 523 : cluster [DBG] osd.6 reported immediately failed by osd.7 2024-02-19T21:58:29.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.688305+0000 mon.smithi131 (mon.0) 524 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:29.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.688457+0000 mon.smithi131 (mon.0) 525 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.688562+0000 mon.smithi131 (mon.0) 526 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.767905+0000 mon.smithi131 (mon.0) 527 : cluster [DBG] osd.6 reported immediately failed by osd.4 2024-02-19T21:58:29.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.811836+0000 mon.smithi131 (mon.0) 528 : cluster [DBG] osd.6 reported immediately failed by osd.5 2024-02-19T21:58:30.127 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.569265+0000 mgr.smithi131.cjubsp (mgr.14720) 214 : cluster [DBG] pgmap v157: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:58:30.127 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.853358+0000 mon.smithi131 (mon.0) 529 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:58:30.127 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:28.862567+0000 mon.smithi131 (mon.0) 530 : cluster [DBG] osdmap e101: 8 total, 7 up, 8 in 2024-02-19T21:58:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.569265+0000 mgr.smithi131.cjubsp (mgr.14720) 214 : cluster [DBG] pgmap v157: 129 pgs: 129 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:58:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.853358+0000 mon.smithi131 (mon.0) 529 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:58:30.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:28.862567+0000 mon.smithi131 (mon.0) 530 : cluster [DBG] osdmap e101: 8 total, 7 up, 8 in 2024-02-19T21:58:30.868 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:30 smithi196 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.6.service: Succeeded. 2024-02-19T21:58:30.868 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:30 smithi196 systemd[1]: Stopped Ceph osd.6 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:58:30.868 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:30 smithi196 systemd[1]: Starting Ceph osd.6 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:58:31.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:29.864391+0000 mon.smithi131 (mon.0) 531 : cluster [DBG] osdmap e102: 8 total, 7 up, 8 in 2024-02-19T21:58:31.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:29.864391+0000 mon.smithi131 (mon.0) 531 : cluster [DBG] osdmap e102: 8 total, 7 up, 8 in 2024-02-19T21:58:31.784 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:31 smithi196 bash[63569]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-02-19T21:58:31.784 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:31 smithi196 bash[63569]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-19T21:58:31.784 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:31 smithi196 bash[63569]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-02-19T21:58:31.784 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:31 smithi196 bash[63569]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-02-19T21:58:31.784 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:31 smithi196 bash[63569]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-02-19T21:58:31.784 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:31 smithi196 bash[63569]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-19T21:58:31.784 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:31 smithi196 bash[63569]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-19T21:58:31.784 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:31 smithi196 bash[63569]: --> ceph-volume lvm activate successful for osd ID: 6 2024-02-19T21:58:32.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:30.569852+0000 mgr.smithi131.cjubsp (mgr.14720) 215 : cluster [DBG] pgmap v160: 129 pgs: 4 active+undersized+degraded, 10 peering, 8 stale+active+clean, 5 active+undersized, 102 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 30/1353 objects degraded (2.217%) 2024-02-19T21:58:32.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:30.861168+0000 mon.smithi131 (mon.0) 532 : cluster [WRN] Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-02-19T21:58:32.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:30.861242+0000 mon.smithi131 (mon.0) 533 : cluster [WRN] Health check failed: Degraded data redundancy: 30/1353 objects degraded (2.217%), 4 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:30.569852+0000 mgr.smithi131.cjubsp (mgr.14720) 215 : cluster [DBG] pgmap v160: 129 pgs: 4 active+undersized+degraded, 10 peering, 8 stale+active+clean, 5 active+undersized, 102 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s; 30/1353 objects degraded (2.217%) 2024-02-19T21:58:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:30.861168+0000 mon.smithi131 (mon.0) 532 : cluster [WRN] Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-02-19T21:58:32.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:30.861242+0000 mon.smithi131 (mon.0) 533 : cluster [WRN] Health check failed: Degraded data redundancy: 30/1353 objects degraded (2.217%), 4 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:32.574 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:32.778 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:58:32.693+0000 7efc3ee7a700 1 -- 172.21.15.196:0/1957268434 <== mon.1 v2:172.21.15.196:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555c20eba680 con 0x555c20e81000 2024-02-19T21:58:33.173 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:32 smithi196 bash[63807]: e9be2f442ef318291e11df730831f0ab7c917762f645f72beef6024a210dd69e 2024-02-19T21:58:33.173 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:32 smithi196 systemd[1]: Started Ceph osd.6 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:58:33.527 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:58:33.208+0000 7efc46908380 -1 Falling back to public interface 2024-02-19T21:58:34.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:32.570541+0000 mgr.smithi131.cjubsp (mgr.14720) 216 : cluster [DBG] pgmap v161: 129 pgs: 18 active+undersized+degraded, 10 peering, 15 active+undersized, 86 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 118/1353 objects degraded (8.721%) 2024-02-19T21:58:34.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:32.827918+0000 mon.smithi131 (mon.0) 534 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:34.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:32.828870+0000 mon.smithi131 (mon.0) 535 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:58:34.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:32.570541+0000 mgr.smithi131.cjubsp (mgr.14720) 216 : cluster [DBG] pgmap v161: 129 pgs: 18 active+undersized+degraded, 10 peering, 15 active+undersized, 86 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 118/1353 objects degraded (8.721%) 2024-02-19T21:58:34.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:32.827918+0000 mon.smithi131 (mon.0) 534 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:34.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:32.828870+0000 mon.smithi131 (mon.0) 535 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:58:35.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:34.571192+0000 mgr.smithi131.cjubsp (mgr.14720) 217 : cluster [DBG] pgmap v162: 129 pgs: 23 active+undersized+degraded, 3 peering, 17 active+undersized, 86 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 144/1353 objects degraded (10.643%) 2024-02-19T21:58:35.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:35.681293+0000 mon.smithi131 (mon.0) 536 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:34.571192+0000 mgr.smithi131.cjubsp (mgr.14720) 217 : cluster [DBG] pgmap v162: 129 pgs: 23 active+undersized+degraded, 3 peering, 17 active+undersized, 86 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 144/1353 objects degraded (10.643%) 2024-02-19T21:58:36.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:35.681293+0000 mon.smithi131 (mon.0) 536 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:37.135 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:36.832239+0000 mon.smithi131 (mon.0) 537 : cluster [WRN] Health check update: Degraded data redundancy: 152/1353 objects degraded (11.234%), 24 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:37.136 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:36.832267+0000 mon.smithi131 (mon.0) 538 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-19T21:58:37.136 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.877197+0000 mon.smithi131 (mon.0) 539 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:37.136 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.878097+0000 mon.smithi131 (mon.0) 540 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:58:37.136 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.878964+0000 mon.smithi131 (mon.0) 541 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:58:37.136 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.883409+0000 mon.smithi131 (mon.0) 542 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:37.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:36.832239+0000 mon.smithi131 (mon.0) 537 : cluster [WRN] Health check update: Degraded data redundancy: 152/1353 objects degraded (11.234%), 24 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:37.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:36.832267+0000 mon.smithi131 (mon.0) 538 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-19T21:58:37.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.877197+0000 mon.smithi131 (mon.0) 539 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:37.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.878097+0000 mon.smithi131 (mon.0) 540 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:58:37.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.878964+0000 mon.smithi131 (mon.0) 541 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:58:37.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.883409+0000 mon.smithi131 (mon.0) 542 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:37.423 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:58:37.135+0000 7efc46908380 -1 osd.6 100 log_to_monitors {default=true} 2024-02-19T21:58:37.614 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:38.173 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:58:37.935+0000 7efc37def700 -1 osd.6 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-19T21:58:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:36.571969+0000 mgr.smithi131.cjubsp (mgr.14720) 218 : cluster [DBG] pgmap v163: 129 pgs: 24 active+undersized+degraded, 19 active+undersized, 86 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 152/1353 objects degraded (11.234%) 2024-02-19T21:58:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.892432+0000 mon.smithi131 (mon.0) 543 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.895871+0000 mon.smithi131 (mon.0) 544 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:58:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.898686+0000 mon.smithi131 (mon.0) 545 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:38.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.900291+0000 mon.smithi131 (mon.0) 546 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:38.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.901878+0000 mon.smithi131 (mon.0) 547 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:38.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.903374+0000 mon.smithi131 (mon.0) 548 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-19T21:58:38.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:36.903539+0000 mgr.smithi131.cjubsp (mgr.14720) 219 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-19T21:58:38.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:58:36.904344+0000 mgr.smithi131.cjubsp (mgr.14720) 220 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-02-19T21:58:38.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:37.140162+0000 mon.smithi196 (mon.1) 6 : audit [INF] from='osd.6 [v2:172.21.15.196:6816/3018290145,v1:172.21.15.196:6817/3018290145]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-19T21:58:38.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:37.140536+0000 mon.smithi131 (mon.0) 549 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:36.571969+0000 mgr.smithi131.cjubsp (mgr.14720) 218 : cluster [DBG] pgmap v163: 129 pgs: 24 active+undersized+degraded, 19 active+undersized, 86 active+clean; 303 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 152/1353 objects degraded (11.234%) 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.892432+0000 mon.smithi131 (mon.0) 543 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.895871+0000 mon.smithi131 (mon.0) 544 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.898686+0000 mon.smithi131 (mon.0) 545 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.900291+0000 mon.smithi131 (mon.0) 546 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.901878+0000 mon.smithi131 (mon.0) 547 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.903374+0000 mon.smithi131 (mon.0) 548 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:36.903539+0000 mgr.smithi131.cjubsp (mgr.14720) 219 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:58:36.904344+0000 mgr.smithi131.cjubsp (mgr.14720) 220 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-02-19T21:58:38.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:37.140162+0000 mon.smithi196 (mon.1) 6 : audit [INF] from='osd.6 [v2:172.21.15.196:6816/3018290145,v1:172.21.15.196:6817/3018290145]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-19T21:58:38.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:37.140536+0000 mon.smithi131 (mon.0) 549 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-19T21:58:38.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:37.925732+0000 mon.smithi131 (mon.0) 550 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-19T21:58:38.937 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:37.925816+0000 mon.smithi131 (mon.0) 551 : cluster [DBG] osdmap e103: 8 total, 7 up, 8 in 2024-02-19T21:58:39.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:37.927303+0000 mon.smithi196 (mon.1) 7 : audit [INF] from='osd.6 [v2:172.21.15.196:6816/3018290145,v1:172.21.15.196:6817/3018290145]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:58:39.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:37.927636+0000 mon.smithi131 (mon.0) 552 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:58:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:37.925732+0000 mon.smithi131 (mon.0) 550 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-19T21:58:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:37.925816+0000 mon.smithi131 (mon.0) 551 : cluster [DBG] osdmap e103: 8 total, 7 up, 8 in 2024-02-19T21:58:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:37.927303+0000 mon.smithi196 (mon.1) 7 : audit [INF] from='osd.6 [v2:172.21.15.196:6816/3018290145,v1:172.21.15.196:6817/3018290145]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:58:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:37.927636+0000 mon.smithi131 (mon.0) 552 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:58:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:38.572524+0000 mgr.smithi131.cjubsp (mgr.14720) 221 : cluster [DBG] pgmap v165: 129 pgs: 24 active+undersized+degraded, 19 active+undersized, 86 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 235 B/s wr, 0 op/s; 152/1353 objects degraded (11.234%) 2024-02-19T21:58:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:38.921565+0000 mon.smithi131 (mon.0) 553 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:58:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:38.928953+0000 mon.smithi131 (mon.0) 554 : cluster [INF] osd.6 [v2:172.21.15.196:6816/3018290145,v1:172.21.15.196:6817/3018290145] boot 2024-02-19T21:58:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:38.929012+0000 mon.smithi131 (mon.0) 555 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-02-19T21:58:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:38.930181+0000 mon.smithi131 (mon.0) 556 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:58:40.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:39.927251+0000 mon.smithi131 (mon.0) 557 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-02-19T21:58:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:38.572524+0000 mgr.smithi131.cjubsp (mgr.14720) 221 : cluster [DBG] pgmap v165: 129 pgs: 24 active+undersized+degraded, 19 active+undersized, 86 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 235 B/s wr, 0 op/s; 152/1353 objects degraded (11.234%) 2024-02-19T21:58:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:38.921565+0000 mon.smithi131 (mon.0) 553 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:58:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:38.928953+0000 mon.smithi131 (mon.0) 554 : cluster [INF] osd.6 [v2:172.21.15.196:6816/3018290145,v1:172.21.15.196:6817/3018290145] boot 2024-02-19T21:58:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:38.929012+0000 mon.smithi131 (mon.0) 555 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-02-19T21:58:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:38.930181+0000 mon.smithi131 (mon.0) 556 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-19T21:58:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:39.927251+0000 mon.smithi131 (mon.0) 557 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-02-19T21:58:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:40.573273+0000 mgr.smithi131.cjubsp (mgr.14720) 222 : cluster [DBG] pgmap v168: 129 pgs: 7 peering, 16 active+undersized+degraded, 13 active+undersized, 93 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 102/1353 objects degraded (7.539%) 2024-02-19T21:58:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:40.864586+0000 mon.smithi131 (mon.0) 558 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:40.573273+0000 mgr.smithi131.cjubsp (mgr.14720) 222 : cluster [DBG] pgmap v168: 129 pgs: 7 peering, 16 active+undersized+degraded, 13 active+undersized, 93 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s; 102/1353 objects degraded (7.539%) 2024-02-19T21:58:42.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:40.864586+0000 mon.smithi131 (mon.0) 558 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:42.653 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:43.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:42.829592+0000 mon.smithi131 (mon.0) 559 : cluster [WRN] Health check update: Degraded data redundancy: 102/1353 objects degraded (7.539%), 16 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:43.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:42.829592+0000 mon.smithi131 (mon.0) 559 : cluster [WRN] Health check update: Degraded data redundancy: 102/1353 objects degraded (7.539%), 16 pgs degraded (PG_DEGRADED) 2024-02-19T21:58:44.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:42.574013+0000 mgr.smithi131.cjubsp (mgr.14720) 223 : cluster [DBG] pgmap v169: 129 pgs: 7 peering, 8 active+undersized+degraded, 5 active+undersized, 109 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 341 B/s wr, 0 op/s; 42/1353 objects degraded (3.104%) 2024-02-19T21:58:44.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:42.574013+0000 mgr.smithi131.cjubsp (mgr.14720) 223 : cluster [DBG] pgmap v169: 129 pgs: 7 peering, 8 active+undersized+degraded, 5 active+undersized, 109 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 341 B/s wr, 0 op/s; 42/1353 objects degraded (3.104%) 2024-02-19T21:58:45.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:44.871010+0000 mon.smithi131 (mon.0) 560 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1353 objects degraded (3.104%), 8 pgs degraded) 2024-02-19T21:58:45.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:44.871040+0000 mon.smithi131 (mon.0) 561 : cluster [INF] Cluster is now healthy 2024-02-19T21:58:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:44.871010+0000 mon.smithi131 (mon.0) 560 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1353 objects degraded (3.104%), 8 pgs degraded) 2024-02-19T21:58:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:44.871040+0000 mon.smithi131 (mon.0) 561 : cluster [INF] Cluster is now healthy 2024-02-19T21:58:46.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:44.574550+0000 mgr.smithi131.cjubsp (mgr.14720) 224 : cluster [DBG] pgmap v170: 129 pgs: 1 peering, 128 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 307 B/s rd, 153 B/s wr, 0 op/s 2024-02-19T21:58:46.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:44.574550+0000 mgr.smithi131.cjubsp (mgr.14720) 224 : cluster [DBG] pgmap v170: 129 pgs: 1 peering, 128 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 307 B/s rd, 153 B/s wr, 0 op/s 2024-02-19T21:58:47.696 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:48.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:46.575250+0000 mgr.smithi131.cjubsp (mgr.14720) 225 : cluster [DBG] pgmap v171: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-02-19T21:58:48.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:46.575250+0000 mgr.smithi131.cjubsp (mgr.14720) 225 : cluster [DBG] pgmap v171: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-02-19T21:58:50.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:48.575903+0000 mgr.smithi131.cjubsp (mgr.14720) 226 : cluster [DBG] pgmap v172: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 212 B/s rd, 212 B/s wr, 0 op/s 2024-02-19T21:58:50.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:48.575903+0000 mgr.smithi131.cjubsp (mgr.14720) 226 : cluster [DBG] pgmap v172: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 212 B/s rd, 212 B/s wr, 0 op/s 2024-02-19T21:58:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:50.576485+0000 mgr.smithi131.cjubsp (mgr.14720) 227 : cluster [DBG] pgmap v173: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 192 B/s rd, 192 B/s wr, 0 op/s 2024-02-19T21:58:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:50.682049+0000 mon.smithi131 (mon.0) 562 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:52.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:50.878634+0000 mon.smithi131 (mon.0) 563 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:50.576485+0000 mgr.smithi131.cjubsp (mgr.14720) 227 : cluster [DBG] pgmap v173: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 192 B/s rd, 192 B/s wr, 0 op/s 2024-02-19T21:58:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:50.682049+0000 mon.smithi131 (mon.0) 562 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:50.878634+0000 mon.smithi131 (mon.0) 563 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:52.737 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:51.905280+0000 mon.smithi131 (mon.0) 564 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-19T21:58:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:51.905499+0000 mgr.smithi131.cjubsp (mgr.14720) 228 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-19T21:58:53.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:58:51.906695+0000 mgr.smithi131.cjubsp (mgr.14720) 229 : cephadm [INF] Upgrade: osd.7 is safe to restart 2024-02-19T21:58:53.014 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:51.905280+0000 mon.smithi131 (mon.0) 564 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-19T21:58:53.014 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:51.905499+0000 mgr.smithi131.cjubsp (mgr.14720) 228 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-19T21:58:53.015 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:58:51.906695+0000 mgr.smithi131.cjubsp (mgr.14720) 229 : cephadm [INF] Upgrade: osd.7 is safe to restart 2024-02-19T21:58:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:52.577189+0000 mgr.smithi131.cjubsp (mgr.14720) 230 : cluster [DBG] pgmap v174: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:58:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:53.549870+0000 mon.smithi131 (mon.0) 565 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:53.550736+0000 mon.smithi131 (mon.0) 566 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-19T21:58:53.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:53.551420+0000 mon.smithi131 (mon.0) 567 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:58:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:52.577189+0000 mgr.smithi131.cjubsp (mgr.14720) 230 : cluster [DBG] pgmap v174: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-19T21:58:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:53.549870+0000 mon.smithi131 (mon.0) 565 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:58:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:53.550736+0000 mon.smithi131 (mon.0) 566 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-19T21:58:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:53.551420+0000 mon.smithi131 (mon.0) 567 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:58:55.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:58:53.544925+0000 mgr.smithi131.cjubsp (mgr.14720) 231 : cephadm [INF] Upgrade: Updating osd.7 2024-02-19T21:58:55.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:58:53.552716+0000 mgr.smithi131.cjubsp (mgr.14720) 232 : cephadm [INF] Deploying daemon osd.7 on smithi196 2024-02-19T21:58:55.026 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:58:53.544925+0000 mgr.smithi131.cjubsp (mgr.14720) 231 : cephadm [INF] Upgrade: Updating osd.7 2024-02-19T21:58:55.026 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:58:53.552716+0000 mgr.smithi131.cjubsp (mgr.14720) 232 : cephadm [INF] Deploying daemon osd.7 on smithi196 2024-02-19T21:58:56.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:54.577874+0000 mgr.smithi131.cjubsp (mgr.14720) 233 : cluster [DBG] pgmap v175: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:58:56.077 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:54.577874+0000 mgr.smithi131.cjubsp (mgr.14720) 233 : cluster [DBG] pgmap v175: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:58:56.440 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:58:56.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:58:56 smithi196 systemd[1]: Stopping Ceph osd.7 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (11m) 106s ago 13m 21.1M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (3m) 106s ago 13m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (3m) 20s ago 12m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (11m) 106s ago 13m 34.6M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (7m) 106s ago 7m 39.0M - 16.2.5 6933c2a0b7dd 56c992d5f64e 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (7m) 20s ago 7m 13.8M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (4m) 106s ago 14m 440M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:58:57.013 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (4m) 20s ago 12m 389M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (4m) 106s ago 14m 48.0M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (4m) 20s ago 12m 50.4M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (5m) 106s ago 5m 80.0M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (5m) 20s ago 5m 77.7M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (12m) 106s ago 12m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (12m) 20s ago 12m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (3m) 106s ago 11m 113M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (2m) 106s ago 11m 101M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (2m) 106s ago 10m 80.6M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (113s) 106s ago 10m 13.3M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (82s) 20s ago 10m 117M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (53s) 20s ago 9m 111M 2712M 16.2.14-548-g81bd20d6 823b921721f5 4c4b41297954 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (24s) 20s ago 9m 10.5M 2712M 16.2.14-548-g81bd20d6 823b921721f5 e9be2f442ef3 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (9m) 20s ago 9m 466M 2712M 16.2.5 6933c2a0b7dd 7eacbdc98fe7 2024-02-19T21:58:57.014 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (4m) 106s ago 12m 60.5M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:58:57.107 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:58:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.7[49192]: debug 2024-02-19T21:58:56.679+0000 7f9e1a592700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T21:58:57.107 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:58:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.7[49192]: debug 2024-02-19T21:58:56.679+0000 7f9e1a592700 -1 osd.7 105 *** Got signal Terminated *** 2024-02-19T21:58:57.107 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:58:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.7[49192]: debug 2024-02-19T21:58:56.679+0000 7f9e1a592700 -1 osd.7 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T21:58:57.352 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:58:57.352 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:58:57.352 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:58:57.352 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:58:57.352 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:58:57.352 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:58:57.352 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:58:57.352 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 7, 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 11, 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 3 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:58:57.353 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:58:57.423 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:58:57 smithi196 bash[65144]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.7 2024-02-19T21:58:57.644 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "13/23 daemons upgraded", 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading osd daemons", 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:58:57.645 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:58:57.775 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:58:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:56.428646+0000 mgr.smithi131.cjubsp (mgr.14720) 234 : audit [DBG] from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.578434+0000 mgr.smithi131.cjubsp (mgr.14720) 235 : cluster [DBG] pgmap v176: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:58:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:56.719973+0000 mgr.smithi131.cjubsp (mgr.14720) 236 : audit [DBG] from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974254+0000 mon.smithi131 (mon.0) 568 : cluster [DBG] osd.7 reported immediately failed by osd.1 2024-02-19T21:58:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974282+0000 mon.smithi131 (mon.0) 569 : cluster [INF] osd.7 failed (root=default,host=smithi196) (connection refused reported by osd.1) 2024-02-19T21:58:58.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974403+0000 mon.smithi131 (mon.0) 570 : cluster [DBG] osd.7 reported immediately failed by osd.1 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974445+0000 mon.smithi131 (mon.0) 571 : cluster [DBG] osd.7 reported immediately failed by osd.2 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974542+0000 mon.smithi131 (mon.0) 572 : cluster [DBG] osd.7 reported immediately failed by osd.3 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974619+0000 mon.smithi131 (mon.0) 573 : cluster [DBG] osd.7 reported immediately failed by osd.4 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974716+0000 mon.smithi131 (mon.0) 574 : cluster [DBG] osd.7 reported immediately failed by osd.0 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974754+0000 mon.smithi131 (mon.0) 575 : cluster [DBG] osd.7 reported immediately failed by osd.2 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974859+0000 mon.smithi131 (mon.0) 576 : cluster [DBG] osd.7 reported immediately failed by osd.3 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.974928+0000 mon.smithi131 (mon.0) 577 : cluster [DBG] osd.7 reported immediately failed by osd.5 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.975026+0000 mon.smithi131 (mon.0) 578 : cluster [DBG] osd.7 reported immediately failed by osd.0 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.975091+0000 mon.smithi131 (mon.0) 579 : cluster [DBG] osd.7 reported immediately failed by osd.6 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.975217+0000 mon.smithi131 (mon.0) 580 : cluster [DBG] osd.7 reported immediately failed by osd.4 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.975317+0000 mon.smithi131 (mon.0) 581 : cluster [DBG] osd.7 reported immediately failed by osd.5 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:56.975385+0000 mon.smithi131 (mon.0) 582 : cluster [DBG] osd.7 reported immediately failed by osd.6 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:57.005044+0000 mgr.smithi131.cjubsp (mgr.14720) 237 : audit [DBG] from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:58.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:57.353105+0000 mon.smithi131 (mon.0) 583 : audit [DBG] from='client.? 172.21.15.131:0/2135992977' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375229+0000 mon.smithi131 (mon.0) 584 : cluster [DBG] osd.7 reported immediately failed by osd.3 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375330+0000 mon.smithi131 (mon.0) 585 : cluster [DBG] osd.7 reported immediately failed by osd.2 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375405+0000 mon.smithi131 (mon.0) 586 : cluster [DBG] osd.7 reported immediately failed by osd.3 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375482+0000 mon.smithi131 (mon.0) 587 : cluster [DBG] osd.7 reported immediately failed by osd.0 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375525+0000 mon.smithi131 (mon.0) 588 : cluster [DBG] osd.7 reported immediately failed by osd.1 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375614+0000 mon.smithi131 (mon.0) 589 : cluster [DBG] osd.7 reported immediately failed by osd.2 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375687+0000 mon.smithi131 (mon.0) 590 : cluster [DBG] osd.7 reported immediately failed by osd.5 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375795+0000 mon.smithi131 (mon.0) 591 : cluster [DBG] osd.7 reported immediately failed by osd.0 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375866+0000 mon.smithi131 (mon.0) 592 : cluster [DBG] osd.7 reported immediately failed by osd.1 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.375963+0000 mon.smithi131 (mon.0) 593 : cluster [DBG] osd.7 reported immediately failed by osd.5 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.376060+0000 mon.smithi131 (mon.0) 594 : cluster [DBG] osd.7 reported immediately failed by osd.6 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.376152+0000 mon.smithi131 (mon.0) 595 : cluster [DBG] osd.7 reported immediately failed by osd.4 2024-02-19T21:58:58.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.376244+0000 mon.smithi131 (mon.0) 596 : cluster [DBG] osd.7 reported immediately failed by osd.6 2024-02-19T21:58:58.009 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.376334+0000 mon.smithi131 (mon.0) 597 : cluster [DBG] osd.7 reported immediately failed by osd.4 2024-02-19T21:58:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:56.428646+0000 mgr.smithi131.cjubsp (mgr.14720) 234 : audit [DBG] from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.578434+0000 mgr.smithi131.cjubsp (mgr.14720) 235 : cluster [DBG] pgmap v176: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-19T21:58:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:56.719973+0000 mgr.smithi131.cjubsp (mgr.14720) 236 : audit [DBG] from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974254+0000 mon.smithi131 (mon.0) 568 : cluster [DBG] osd.7 reported immediately failed by osd.1 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974282+0000 mon.smithi131 (mon.0) 569 : cluster [INF] osd.7 failed (root=default,host=smithi196) (connection refused reported by osd.1) 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974403+0000 mon.smithi131 (mon.0) 570 : cluster [DBG] osd.7 reported immediately failed by osd.1 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974445+0000 mon.smithi131 (mon.0) 571 : cluster [DBG] osd.7 reported immediately failed by osd.2 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974542+0000 mon.smithi131 (mon.0) 572 : cluster [DBG] osd.7 reported immediately failed by osd.3 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974619+0000 mon.smithi131 (mon.0) 573 : cluster [DBG] osd.7 reported immediately failed by osd.4 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974716+0000 mon.smithi131 (mon.0) 574 : cluster [DBG] osd.7 reported immediately failed by osd.0 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974754+0000 mon.smithi131 (mon.0) 575 : cluster [DBG] osd.7 reported immediately failed by osd.2 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974859+0000 mon.smithi131 (mon.0) 576 : cluster [DBG] osd.7 reported immediately failed by osd.3 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.974928+0000 mon.smithi131 (mon.0) 577 : cluster [DBG] osd.7 reported immediately failed by osd.5 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.975026+0000 mon.smithi131 (mon.0) 578 : cluster [DBG] osd.7 reported immediately failed by osd.0 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.975091+0000 mon.smithi131 (mon.0) 579 : cluster [DBG] osd.7 reported immediately failed by osd.6 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.975217+0000 mon.smithi131 (mon.0) 580 : cluster [DBG] osd.7 reported immediately failed by osd.4 2024-02-19T21:58:58.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.975317+0000 mon.smithi131 (mon.0) 581 : cluster [DBG] osd.7 reported immediately failed by osd.5 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:56.975385+0000 mon.smithi131 (mon.0) 582 : cluster [DBG] osd.7 reported immediately failed by osd.6 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:57.005044+0000 mgr.smithi131.cjubsp (mgr.14720) 237 : audit [DBG] from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:57.353105+0000 mon.smithi131 (mon.0) 583 : audit [DBG] from='client.? 172.21.15.131:0/2135992977' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375229+0000 mon.smithi131 (mon.0) 584 : cluster [DBG] osd.7 reported immediately failed by osd.3 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375330+0000 mon.smithi131 (mon.0) 585 : cluster [DBG] osd.7 reported immediately failed by osd.2 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375405+0000 mon.smithi131 (mon.0) 586 : cluster [DBG] osd.7 reported immediately failed by osd.3 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375482+0000 mon.smithi131 (mon.0) 587 : cluster [DBG] osd.7 reported immediately failed by osd.0 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375525+0000 mon.smithi131 (mon.0) 588 : cluster [DBG] osd.7 reported immediately failed by osd.1 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375614+0000 mon.smithi131 (mon.0) 589 : cluster [DBG] osd.7 reported immediately failed by osd.2 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375687+0000 mon.smithi131 (mon.0) 590 : cluster [DBG] osd.7 reported immediately failed by osd.5 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375795+0000 mon.smithi131 (mon.0) 591 : cluster [DBG] osd.7 reported immediately failed by osd.0 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375866+0000 mon.smithi131 (mon.0) 592 : cluster [DBG] osd.7 reported immediately failed by osd.1 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.375963+0000 mon.smithi131 (mon.0) 593 : cluster [DBG] osd.7 reported immediately failed by osd.5 2024-02-19T21:58:58.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.376060+0000 mon.smithi131 (mon.0) 594 : cluster [DBG] osd.7 reported immediately failed by osd.6 2024-02-19T21:58:58.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.376152+0000 mon.smithi131 (mon.0) 595 : cluster [DBG] osd.7 reported immediately failed by osd.4 2024-02-19T21:58:58.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.376244+0000 mon.smithi131 (mon.0) 596 : cluster [DBG] osd.7 reported immediately failed by osd.6 2024-02-19T21:58:58.176 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.376334+0000 mon.smithi131 (mon.0) 597 : cluster [DBG] osd.7 reported immediately failed by osd.4 2024-02-19T21:58:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:58:57.645539+0000 mgr.smithi131.cjubsp (mgr.14720) 238 : audit [DBG] from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.697627+0000 mon.smithi131 (mon.0) 598 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:58:59.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:57.708652+0000 mon.smithi131 (mon.0) 599 : cluster [DBG] osdmap e106: 8 total, 7 up, 8 in 2024-02-19T21:58:59.164 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:58:57.645539+0000 mgr.smithi131.cjubsp (mgr.14720) 238 : audit [DBG] from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:58:59.164 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.697627+0000 mon.smithi131 (mon.0) 598 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-02-19T21:58:59.164 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:57.708652+0000 mon.smithi131 (mon.0) 599 : cluster [DBG] osdmap e106: 8 total, 7 up, 8 in 2024-02-19T21:58:59.915 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:58.579142+0000 mgr.smithi131.cjubsp (mgr.14720) 239 : cluster [DBG] pgmap v178: 129 pgs: 8 peering, 15 stale+active+clean, 106 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:58:59.915 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:58.706384+0000 mon.smithi131 (mon.0) 600 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-19T21:58:59.915 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:58:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:58:58.711973+0000 mon.smithi131 (mon.0) 601 : cluster [DBG] osdmap e107: 8 total, 7 up, 8 in 2024-02-19T21:58:59.916 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:58:59 smithi196 systemd[1]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.7.service: Succeeded. 2024-02-19T21:58:59.916 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:58:59 smithi196 systemd[1]: Stopped Ceph osd.7 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:58:59.916 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:58:59 smithi196 systemd[1]: Starting Ceph osd.7 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T21:59:00.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:58.579142+0000 mgr.smithi131.cjubsp (mgr.14720) 239 : cluster [DBG] pgmap v178: 129 pgs: 8 peering, 15 stale+active+clean, 106 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-02-19T21:59:00.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:58.706384+0000 mon.smithi131 (mon.0) 600 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-19T21:59:00.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:58:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:58:58.711973+0000 mon.smithi131 (mon.0) 601 : cluster [DBG] osdmap e107: 8 total, 7 up, 8 in 2024-02-19T21:59:00.675 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:00 smithi196 bash[65439]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-02-19T21:59:00.675 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:00 smithi196 bash[65439]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-19T21:59:00.675 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:00 smithi196 bash[65439]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-02-19T21:59:00.675 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:00 smithi196 bash[65439]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-02-19T21:59:00.676 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:00 smithi196 bash[65439]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-02-19T21:59:00.676 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:00 smithi196 bash[65439]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-19T21:59:00.676 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:00 smithi196 bash[65439]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-19T21:59:00.676 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:00 smithi196 bash[65439]: --> ceph-volume lvm activate successful for osd ID: 7 2024-02-19T21:59:01.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:00.703385+0000 mon.smithi131 (mon.0) 602 : cluster [WRN] Health check failed: Degraded data redundancy: 93/1353 objects degraded (6.874%), 17 pgs degraded (PG_DEGRADED) 2024-02-19T21:59:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:00.703385+0000 mon.smithi131 (mon.0) 602 : cluster [WRN] Health check failed: Degraded data redundancy: 93/1353 objects degraded (6.874%), 17 pgs degraded (PG_DEGRADED) 2024-02-19T21:59:01.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T21:59:01.567+0000 7ff665104700 1 -- 172.21.15.196:0/2207726442 <== mon.1 v2:172.21.15.196:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ecc1ec2680 con 0x55ecc1e89000 2024-02-19T21:59:01.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:01 smithi196 bash[65701]: 10d1c2418ed858b9099a254c4a21814b49f0e7dec43e9564a7c922e65e447705 2024-02-19T21:59:01.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:01 smithi196 systemd[1]: Started Ceph osd.7 for e8e4a71a-cf6f-11ee-95bb-87774f69a715. 2024-02-19T21:59:01.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:00.579838+0000 mgr.smithi131.cjubsp (mgr.14720) 240 : cluster [DBG] pgmap v180: 129 pgs: 17 active+undersized+degraded, 8 peering, 9 stale+active+clean, 11 active+undersized, 84 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 93/1353 objects degraded (6.874%) 2024-02-19T21:59:01.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:01.721465+0000 mon.smithi131 (mon.0) 603 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:01.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:01.722393+0000 mon.smithi131 (mon.0) 604 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:00.579838+0000 mgr.smithi131.cjubsp (mgr.14720) 240 : cluster [DBG] pgmap v180: 129 pgs: 17 active+undersized+degraded, 8 peering, 9 stale+active+clean, 11 active+undersized, 84 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 127 B/s wr, 0 op/s; 93/1353 objects degraded (6.874%) 2024-02-19T21:59:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:01.721465+0000 mon.smithi131 (mon.0) 603 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:02.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:01.722393+0000 mon.smithi131 (mon.0) 604 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:02.417 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T21:59:02.083+0000 7ff66cb92380 -1 Falling back to public interface 2024-02-19T21:59:02.808 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:59:04.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:02.580559+0000 mgr.smithi131.cjubsp (mgr.14720) 241 : cluster [DBG] pgmap v181: 129 pgs: 31 active+undersized+degraded, 8 peering, 17 active+undersized, 73 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 193/1353 objects degraded (14.265%) 2024-02-19T21:59:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:02.580559+0000 mgr.smithi131.cjubsp (mgr.14720) 241 : cluster [DBG] pgmap v181: 129 pgs: 31 active+undersized+degraded, 8 peering, 17 active+undersized, 73 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 193/1353 objects degraded (14.265%) 2024-02-19T21:59:05.121 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:04.759612+0000 mon.smithi131 (mon.0) 605 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-19T21:59:05.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:04.759612+0000 mon.smithi131 (mon.0) 605 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-19T21:59:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:04.581273+0000 mgr.smithi131.cjubsp (mgr.14720) 242 : cluster [DBG] pgmap v182: 129 pgs: 36 active+undersized+degraded, 20 active+undersized, 73 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 210/1353 objects degraded (15.521%) 2024-02-19T21:59:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.598805+0000 mon.smithi131 (mon.0) 606 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.682219+0000 mon.smithi131 (mon.0) 607 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:04.581273+0000 mgr.smithi131.cjubsp (mgr.14720) 242 : cluster [DBG] pgmap v182: 129 pgs: 36 active+undersized+degraded, 20 active+undersized, 73 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 210/1353 objects degraded (15.521%) 2024-02-19T21:59:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.598805+0000 mon.smithi131 (mon.0) 606 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:06.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.682219+0000 mon.smithi131 (mon.0) 607 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.881931+0000 mon.smithi131 (mon.0) 608 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.882886+0000 mon.smithi131 (mon.0) 609 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.888092+0000 mon.smithi131 (mon.0) 610 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.895994+0000 mon.smithi131 (mon.0) 611 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.899253+0000 mon.smithi131 (mon.0) 612 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.901774+0000 mon.smithi131 (mon.0) 613 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.903079+0000 mon.smithi131 (mon.0) 614 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.904443+0000 mon.smithi131 (mon.0) 615 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.906303+0000 mon.smithi131 (mon.0) 616 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:05.906981+0000 mgr.smithi131.cjubsp (mgr.14720) 243 : cephadm [INF] Upgrade: Setting container_image for all osd 2024-02-19T21:59:07.112 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.910885+0000 mon.smithi131 (mon.0) 617 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.911607+0000 mon.smithi131 (mon.0) 618 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.914729+0000 mon.smithi131 (mon.0) 619 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.915359+0000 mon.smithi131 (mon.0) 620 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.918391+0000 mon.smithi131 (mon.0) 621 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.919037+0000 mon.smithi131 (mon.0) 622 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.922102+0000 mon.smithi131 (mon.0) 623 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.922671+0000 mon.smithi131 (mon.0) 624 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.925744+0000 mon.smithi131 (mon.0) 625 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.926352+0000 mon.smithi131 (mon.0) 626 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.929359+0000 mon.smithi131 (mon.0) 627 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.930001+0000 mon.smithi131 (mon.0) 628 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.935491+0000 mon.smithi131 (mon.0) 629 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.936094+0000 mon.smithi131 (mon.0) 630 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-19T21:59:07.113 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.941485+0000 mon.smithi131 (mon.0) 631 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-02-19T21:59:07.114 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.942114+0000 mon.smithi131 (mon.0) 632 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-19T21:59:07.114 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.947410+0000 mon.smithi131 (mon.0) 633 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-02-19T21:59:07.114 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:05.949236+0000 mon.smithi131 (mon.0) 634 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi131.gemwnz"]}]: dispatch 2024-02-19T21:59:07.114 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:05.949741+0000 mgr.smithi131.cjubsp (mgr.14720) 244 : cephadm [INF] Upgrade: It appears safe to stop mds.foofs.smithi131.gemwnz 2024-02-19T21:59:07.124 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.881931+0000 mon.smithi131 (mon.0) 608 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:07.124 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.882886+0000 mon.smithi131 (mon.0) 609 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.888092+0000 mon.smithi131 (mon.0) 610 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.895994+0000 mon.smithi131 (mon.0) 611 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.899253+0000 mon.smithi131 (mon.0) 612 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.901774+0000 mon.smithi131 (mon.0) 613 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.903079+0000 mon.smithi131 (mon.0) 614 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.904443+0000 mon.smithi131 (mon.0) 615 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.906303+0000 mon.smithi131 (mon.0) 616 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:05.906981+0000 mgr.smithi131.cjubsp (mgr.14720) 243 : cephadm [INF] Upgrade: Setting container_image for all osd 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.910885+0000 mon.smithi131 (mon.0) 617 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.911607+0000 mon.smithi131 (mon.0) 618 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.914729+0000 mon.smithi131 (mon.0) 619 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.915359+0000 mon.smithi131 (mon.0) 620 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.918391+0000 mon.smithi131 (mon.0) 621 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-02-19T21:59:07.125 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.919037+0000 mon.smithi131 (mon.0) 622 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.922102+0000 mon.smithi131 (mon.0) 623 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.922671+0000 mon.smithi131 (mon.0) 624 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.925744+0000 mon.smithi131 (mon.0) 625 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.926352+0000 mon.smithi131 (mon.0) 626 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.929359+0000 mon.smithi131 (mon.0) 627 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.930001+0000 mon.smithi131 (mon.0) 628 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.935491+0000 mon.smithi131 (mon.0) 629 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.936094+0000 mon.smithi131 (mon.0) 630 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.941485+0000 mon.smithi131 (mon.0) 631 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.942114+0000 mon.smithi131 (mon.0) 632 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.947410+0000 mon.smithi131 (mon.0) 633 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:05.949236+0000 mon.smithi131 (mon.0) 634 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi131.gemwnz"]}]: dispatch 2024-02-19T21:59:07.126 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:05.949741+0000 mgr.smithi131.cjubsp (mgr.14720) 244 : cephadm [INF] Upgrade: It appears safe to stop mds.foofs.smithi131.gemwnz 2024-02-19T21:59:07.423 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T21:59:07.111+0000 7ff66cb92380 -1 osd.7 105 log_to_monitors {default=true} 2024-02-19T21:59:07.846 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:59:08.182 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:06.581962+0000 mgr.smithi131.cjubsp (mgr.14720) 245 : cluster [DBG] pgmap v183: 129 pgs: 36 active+undersized+degraded, 20 active+undersized, 73 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 115 B/s wr, 0 op/s; 210/1353 objects degraded (15.521%) 2024-02-19T21:59:08.182 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:07.115479+0000 mon.smithi196 (mon.1) 8 : audit [INF] from='osd.7 [v2:172.21.15.196:6824/2179038125,v1:172.21.15.196:6825/2179038125]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-19T21:59:08.182 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:07.116054+0000 mon.smithi131 (mon.0) 635 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-19T21:59:08.182 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:07.479031+0000 mon.smithi131 (mon.0) 636 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:08.183 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:07.492043+0000 mon.smithi131 (mon.0) 637 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi131.gemwnz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-19T21:59:08.183 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:07.492655+0000 mon.smithi131 (mon.0) 638 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:08.183 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:07.834438+0000 mon.smithi131 (mon.0) 639 : cluster [WRN] Health check update: Degraded data redundancy: 210/1353 objects degraded (15.521%), 36 pgs degraded (PG_DEGRADED) 2024-02-19T21:59:08.423 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T21:59:07.973+0000 7ff65e87a700 -1 osd.7 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-19T21:59:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:06.581962+0000 mgr.smithi131.cjubsp (mgr.14720) 245 : cluster [DBG] pgmap v183: 129 pgs: 36 active+undersized+degraded, 20 active+undersized, 73 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 115 B/s wr, 0 op/s; 210/1353 objects degraded (15.521%) 2024-02-19T21:59:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:07.115479+0000 mon.smithi196 (mon.1) 8 : audit [INF] from='osd.7 [v2:172.21.15.196:6824/2179038125,v1:172.21.15.196:6825/2179038125]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-19T21:59:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:07.116054+0000 mon.smithi131 (mon.0) 635 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-19T21:59:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:07.479031+0000 mon.smithi131 (mon.0) 636 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:07.492043+0000 mon.smithi131 (mon.0) 637 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi131.gemwnz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-19T21:59:08.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:07.492655+0000 mon.smithi131 (mon.0) 638 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:08.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:07.834438+0000 mon.smithi131 (mon.0) 639 : cluster [WRN] Health check update: Degraded data redundancy: 210/1353 objects degraded (15.521%), 36 pgs degraded (PG_DEGRADED) 2024-02-19T21:59:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:07.474669+0000 mgr.smithi131.cjubsp (mgr.14720) 246 : cephadm [INF] Upgrade: Updating mds.foofs.smithi131.gemwnz 2024-02-19T21:59:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:07.493190+0000 mgr.smithi131.cjubsp (mgr.14720) 247 : cephadm [INF] Deploying daemon mds.foofs.smithi131.gemwnz on smithi131 2024-02-19T21:59:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:07.957831+0000 mon.smithi131 (mon.0) 640 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-19T21:59:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:07.957909+0000 mon.smithi131 (mon.0) 641 : cluster [DBG] osdmap e108: 8 total, 7 up, 8 in 2024-02-19T21:59:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:07.962417+0000 mon.smithi196 (mon.1) 9 : audit [INF] from='osd.7 [v2:172.21.15.196:6824/2179038125,v1:172.21.15.196:6825/2179038125]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:59:09.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:07.962891+0000 mon.smithi131 (mon.0) 642 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:59:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:07.474669+0000 mgr.smithi131.cjubsp (mgr.14720) 246 : cephadm [INF] Upgrade: Updating mds.foofs.smithi131.gemwnz 2024-02-19T21:59:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:07.493190+0000 mgr.smithi131.cjubsp (mgr.14720) 247 : cephadm [INF] Deploying daemon mds.foofs.smithi131.gemwnz on smithi131 2024-02-19T21:59:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:07.957831+0000 mon.smithi131 (mon.0) 640 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-19T21:59:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:07.957909+0000 mon.smithi131 (mon.0) 641 : cluster [DBG] osdmap e108: 8 total, 7 up, 8 in 2024-02-19T21:59:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:07.962417+0000 mon.smithi196 (mon.1) 9 : audit [INF] from='osd.7 [v2:172.21.15.196:6824/2179038125,v1:172.21.15.196:6825/2179038125]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:59:09.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:07.962891+0000 mon.smithi131 (mon.0) 642 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-02-19T21:59:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:59:09.959+0000 7fbb0cfd7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:59:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:08.582442+0000 mgr.smithi131.cjubsp (mgr.14720) 248 : cluster [DBG] pgmap v185: 129 pgs: 36 active+undersized+degraded, 20 active+undersized, 73 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 207 B/s wr, 0 op/s; 210/1353 objects degraded (15.521%) 2024-02-19T21:59:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:08.956093+0000 mon.smithi131 (mon.0) 643 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:59:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:08.962892+0000 mon.smithi131 (mon.0) 644 : cluster [INF] osd.7 [v2:172.21.15.196:6824/2179038125,v1:172.21.15.196:6825/2179038125] boot 2024-02-19T21:59:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:08.962942+0000 mon.smithi131 (mon.0) 645 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-02-19T21:59:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:08.963751+0000 mon.smithi131 (mon.0) 646 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:59:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.269315+0000 mon.smithi131 (mon.0) 647 : cluster [WRN] daemon mds.foofs.smithi131.gemwnz compat changed unexpectedly 2024-02-19T21:59:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.275605+0000 mon.smithi131 (mon.0) 648 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-02-19T21:59:10.423 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:59:09.979+0000 7efc1d503700 -1 bdev(0x555c21bfa800 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 5 2024-02-19T21:59:10.423 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:59:09.979+0000 7efc1acfe700 -1 bdev(0x555c21bfa800 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 7 2024-02-19T21:59:10.423 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:59:09.981+0000 7efc20509700 -1 bdev(0x555c21bfa800 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 11 2024-02-19T21:59:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:08.582442+0000 mgr.smithi131.cjubsp (mgr.14720) 248 : cluster [DBG] pgmap v185: 129 pgs: 36 active+undersized+degraded, 20 active+undersized, 73 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 207 B/s wr, 0 op/s; 210/1353 objects degraded (15.521%) 2024-02-19T21:59:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:08.956093+0000 mon.smithi131 (mon.0) 643 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-19T21:59:10.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:08.962892+0000 mon.smithi131 (mon.0) 644 : cluster [INF] osd.7 [v2:172.21.15.196:6824/2179038125,v1:172.21.15.196:6825/2179038125] boot 2024-02-19T21:59:10.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:08.962942+0000 mon.smithi131 (mon.0) 645 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-02-19T21:59:10.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:08.963751+0000 mon.smithi131 (mon.0) 646 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-19T21:59:10.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.269315+0000 mon.smithi131 (mon.0) 647 : cluster [WRN] daemon mds.foofs.smithi131.gemwnz compat changed unexpectedly 2024-02-19T21:59:10.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.275605+0000 mon.smithi131 (mon.0) 648 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-02-19T21:59:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.960364+0000 mon.smithi131 (mon.0) 649 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-02-19T21:59:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.960412+0000 mon.smithi131 (mon.0) 650 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:59:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.968818+0000 mon.smithi131 (mon.0) 651 : cluster [INF] Standby daemon mds.foofs.smithi196.hhhkwt assigned to filesystem foofs as rank 0 2024-02-19T21:59:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.968926+0000 mon.smithi131 (mon.0) 652 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-02-19T21:59:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.968951+0000 mon.smithi131 (mon.0) 653 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-19T21:59:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.969528+0000 mon.smithi131 (mon.0) 654 : cluster [DBG] fsmap foofs:0/1 1 up:standby, 1 failed 2024-02-19T21:59:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:09.971906+0000 mon.smithi131 (mon.0) 655 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi196.hhhkwt=up:replay} 2024-02-19T21:59:11.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:10.894618+0000 mon.smithi131 (mon.0) 656 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.960364+0000 mon.smithi131 (mon.0) 649 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-02-19T21:59:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.960412+0000 mon.smithi131 (mon.0) 650 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:59:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.968818+0000 mon.smithi131 (mon.0) 651 : cluster [INF] Standby daemon mds.foofs.smithi196.hhhkwt assigned to filesystem foofs as rank 0 2024-02-19T21:59:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.968926+0000 mon.smithi131 (mon.0) 652 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-02-19T21:59:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.968951+0000 mon.smithi131 (mon.0) 653 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-19T21:59:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.969528+0000 mon.smithi131 (mon.0) 654 : cluster [DBG] fsmap foofs:0/1 1 up:standby, 1 failed 2024-02-19T21:59:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:09.971906+0000 mon.smithi131 (mon.0) 655 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi196.hhhkwt=up:replay} 2024-02-19T21:59:11.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:10.894618+0000 mon.smithi131 (mon.0) 656 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:10.583339+0000 mgr.smithi131.cjubsp (mgr.14720) 249 : cluster [DBG] pgmap v188: 129 pgs: 9 peering, 24 active+undersized+degraded, 10 active+undersized, 86 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s rd, 170 B/s wr, 1 op/s; 145/1353 objects degraded (10.717%) 2024-02-19T21:59:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:10.975691+0000 mon.smithi131 (mon.0) 657 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:reconnect 2024-02-19T21:59:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:10.975775+0000 mon.smithi131 (mon.0) 658 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi196.hhhkwt=up:reconnect} 2024-02-19T21:59:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:10.583339+0000 mgr.smithi131.cjubsp (mgr.14720) 249 : cluster [DBG] pgmap v188: 129 pgs: 9 peering, 24 active+undersized+degraded, 10 active+undersized, 86 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s rd, 170 B/s wr, 1 op/s; 145/1353 objects degraded (10.717%) 2024-02-19T21:59:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:10.975691+0000 mon.smithi131 (mon.0) 657 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:reconnect 2024-02-19T21:59:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:10.975775+0000 mon.smithi131 (mon.0) 658 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi196.hhhkwt=up:reconnect} 2024-02-19T21:59:12.895 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:59:13.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:11.981421+0000 mon.smithi131 (mon.0) 659 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:rejoin 2024-02-19T21:59:13.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:11.981476+0000 mon.smithi131 (mon.0) 660 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi196.hhhkwt=up:rejoin} 2024-02-19T21:59:13.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:11.984270+0000 mon.smithi131 (mon.0) 661 : cluster [INF] daemon mds.foofs.smithi196.hhhkwt is now active in filesystem foofs as rank 0 2024-02-19T21:59:13.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:12.835302+0000 mon.smithi131 (mon.0) 662 : cluster [WRN] Health check update: Degraded data redundancy: 145/1353 objects degraded (10.717%), 24 pgs degraded (PG_DEGRADED) 2024-02-19T21:59:13.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:11.981421+0000 mon.smithi131 (mon.0) 659 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:rejoin 2024-02-19T21:59:13.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:11.981476+0000 mon.smithi131 (mon.0) 660 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi196.hhhkwt=up:rejoin} 2024-02-19T21:59:13.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:11.984270+0000 mon.smithi131 (mon.0) 661 : cluster [INF] daemon mds.foofs.smithi196.hhhkwt is now active in filesystem foofs as rank 0 2024-02-19T21:59:13.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:12.835302+0000 mon.smithi131 (mon.0) 662 : cluster [WRN] Health check update: Degraded data redundancy: 145/1353 objects degraded (10.717%), 24 pgs degraded (PG_DEGRADED) 2024-02-19T21:59:14.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:12.584026+0000 mgr.smithi131.cjubsp (mgr.14720) 250 : cluster [DBG] pgmap v189: 129 pgs: 9 peering, 16 active+undersized+degraded, 7 active+undersized, 97 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 341 B/s wr, 5 op/s; 104/1353 objects degraded (7.687%) 2024-02-19T21:59:14.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:12.980172+0000 mon.smithi131 (mon.0) 663 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-02-19T21:59:14.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:12.987245+0000 mon.smithi131 (mon.0) 664 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:active 2024-02-19T21:59:14.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:12.987319+0000 mon.smithi131 (mon.0) 665 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi196.hhhkwt=up:active} 2024-02-19T21:59:14.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:12.584026+0000 mgr.smithi131.cjubsp (mgr.14720) 250 : cluster [DBG] pgmap v189: 129 pgs: 9 peering, 16 active+undersized+degraded, 7 active+undersized, 97 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 341 B/s wr, 5 op/s; 104/1353 objects degraded (7.687%) 2024-02-19T21:59:14.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:12.980172+0000 mon.smithi131 (mon.0) 663 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-02-19T21:59:14.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:12.987245+0000 mon.smithi131 (mon.0) 664 : cluster [DBG] mds.? [v2:172.21.15.196:6832/1671017947,v1:172.21.15.196:6833/1671017947] up:active 2024-02-19T21:59:14.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:12.987319+0000 mon.smithi131 (mon.0) 665 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi196.hhhkwt=up:active} 2024-02-19T21:59:15.377 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:14.986300+0000 mon.smithi131 (mon.0) 666 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 104/1353 objects degraded (7.687%), 16 pgs degraded) 2024-02-19T21:59:15.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:14.986300+0000 mon.smithi131 (mon.0) 666 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 104/1353 objects degraded (7.687%), 16 pgs degraded) 2024-02-19T21:59:16.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:14.584707+0000 mgr.smithi131.cjubsp (mgr.14720) 251 : cluster [DBG] pgmap v190: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 154 B/s wr, 9 op/s 2024-02-19T21:59:16.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:14.584707+0000 mgr.smithi131.cjubsp (mgr.14720) 251 : cluster [DBG] pgmap v190: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 154 B/s wr, 9 op/s 2024-02-19T21:59:17.928 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:59:18.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:16.585254+0000 mgr.smithi131.cjubsp (mgr.14720) 252 : cluster [DBG] pgmap v191: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 127 B/s wr, 9 op/s 2024-02-19T21:59:18.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:16.794476+0000 mon.smithi131 (mon.0) 667 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:18.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:16.795237+0000 mon.smithi131 (mon.0) 668 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:18.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:17.040287+0000 mon.smithi131 (mon.0) 669 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-02-19T21:59:18.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:17.040312+0000 mon.smithi131 (mon.0) 670 : cluster [INF] Cluster is now healthy 2024-02-19T21:59:18.174 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:17.051296+0000 mon.smithi131 (mon.0) 671 : cluster [DBG] mds.? [v2:172.21.15.131:6834/158099597,v1:172.21.15.131:6835/158099597] up:boot 2024-02-19T21:59:18.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:17.051339+0000 mon.smithi131 (mon.0) 672 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi196.hhhkwt=up:active} 1 up:standby 2024-02-19T21:59:18.175 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:17.051779+0000 mon.smithi131 (mon.0) 673 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:59:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:16.585254+0000 mgr.smithi131.cjubsp (mgr.14720) 252 : cluster [DBG] pgmap v191: 129 pgs: 129 active+clean; 303 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 127 B/s wr, 9 op/s 2024-02-19T21:59:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:16.794476+0000 mon.smithi131 (mon.0) 667 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:16.795237+0000 mon.smithi131 (mon.0) 668 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:17.040287+0000 mon.smithi131 (mon.0) 669 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-02-19T21:59:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:17.040312+0000 mon.smithi131 (mon.0) 670 : cluster [INF] Cluster is now healthy 2024-02-19T21:59:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:17.051296+0000 mon.smithi131 (mon.0) 671 : cluster [DBG] mds.? [v2:172.21.15.131:6834/158099597,v1:172.21.15.131:6835/158099597] up:boot 2024-02-19T21:59:18.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:17.051339+0000 mon.smithi131 (mon.0) 672 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi196.hhhkwt=up:active} 1 up:standby 2024-02-19T21:59:18.257 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:17.051779+0000 mon.smithi131 (mon.0) 673 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:59:20.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:19 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:18.585950+0000 mgr.smithi131.cjubsp (mgr.14720) 253 : cluster [DBG] pgmap v192: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 7.2 KiB/s wr, 14 op/s 2024-02-19T21:59:20.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:19 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:18.585950+0000 mgr.smithi131.cjubsp (mgr.14720) 253 : cluster [DBG] pgmap v192: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 21 MiB/s rd, 7.2 KiB/s wr, 14 op/s 2024-02-19T21:59:22.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:20.586480+0000 mgr.smithi131.cjubsp (mgr.14720) 254 : cluster [DBG] pgmap v193: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 6.1 KiB/s wr, 11 op/s 2024-02-19T21:59:22.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:20.682107+0000 mon.smithi131 (mon.0) 674 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:22.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:21 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:20.905163+0000 mon.smithi131 (mon.0) 675 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:20.586480+0000 mgr.smithi131.cjubsp (mgr.14720) 254 : cluster [DBG] pgmap v193: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 6.1 KiB/s wr, 11 op/s 2024-02-19T21:59:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:20.682107+0000 mon.smithi131 (mon.0) 674 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:22.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:21 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:20.905163+0000 mon.smithi131 (mon.0) 675 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:22.984 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:59:23.354 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.255279+0000 mon.smithi131 (mon.0) 676 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:23.354 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.540903+0000 mon.smithi131 (mon.0) 677 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:23.354 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.541545+0000 mon.smithi131 (mon.0) 678 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:59:23.354 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.547508+0000 mon.smithi131 (mon.0) 679 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:23.354 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.552340+0000 mon.smithi131 (mon.0) 680 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:23.355 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.555306+0000 mon.smithi131 (mon.0) 681 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:23.355 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.557391+0000 mon.smithi131 (mon.0) 682 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:23.355 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.558624+0000 mon.smithi131 (mon.0) 683 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:23.355 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.559848+0000 mon.smithi131 (mon.0) 684 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:23.355 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.561730+0000 mon.smithi131 (mon.0) 685 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:23.355 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:23 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:22.563384+0000 mon.smithi131 (mon.0) 686 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi196.hhhkwt"]}]: dispatch 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.255279+0000 mon.smithi131 (mon.0) 676 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.540903+0000 mon.smithi131 (mon.0) 677 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.541545+0000 mon.smithi131 (mon.0) 678 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.547508+0000 mon.smithi131 (mon.0) 679 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.552340+0000 mon.smithi131 (mon.0) 680 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.555306+0000 mon.smithi131 (mon.0) 681 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.557391+0000 mon.smithi131 (mon.0) 682 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.558624+0000 mon.smithi131 (mon.0) 683 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.559848+0000 mon.smithi131 (mon.0) 684 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:23.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.561730+0000 mon.smithi131 (mon.0) 685 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:23.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:23 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:22.563384+0000 mon.smithi131 (mon.0) 686 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi196.hhhkwt"]}]: dispatch 2024-02-19T21:59:24.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:22.563856+0000 mgr.smithi131.cjubsp (mgr.14720) 255 : cephadm [INF] Upgrade: It appears safe to stop mds.foofs.smithi196.hhhkwt 2024-02-19T21:59:24.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:22.587084+0000 mgr.smithi131.cjubsp (mgr.14720) 256 : cluster [DBG] pgmap v194: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 5.8 KiB/s wr, 11 op/s 2024-02-19T21:59:24.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:24.076805+0000 mon.smithi131 (mon.0) 687 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:24.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:24.077681+0000 mon.smithi131 (mon.0) 688 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi196.hhhkwt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-19T21:59:24.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:24.078448+0000 mon.smithi131 (mon.0) 689 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:24.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:22.563856+0000 mgr.smithi131.cjubsp (mgr.14720) 255 : cephadm [INF] Upgrade: It appears safe to stop mds.foofs.smithi196.hhhkwt 2024-02-19T21:59:24.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:22.587084+0000 mgr.smithi131.cjubsp (mgr.14720) 256 : cluster [DBG] pgmap v194: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 5.8 KiB/s wr, 11 op/s 2024-02-19T21:59:24.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:24.076805+0000 mon.smithi131 (mon.0) 687 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:24.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:24.077681+0000 mon.smithi131 (mon.0) 688 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi196.hhhkwt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-19T21:59:24.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:24.078448+0000 mon.smithi131 (mon.0) 689 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:25.592 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:24.072086+0000 mgr.smithi131.cjubsp (mgr.14720) 257 : cephadm [INF] Upgrade: Updating mds.foofs.smithi196.hhhkwt 2024-02-19T21:59:25.592 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:24.079096+0000 mgr.smithi131.cjubsp (mgr.14720) 258 : cephadm [INF] Deploying daemon mds.foofs.smithi196.hhhkwt on smithi196 2024-02-19T21:59:25.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:24.072086+0000 mgr.smithi131.cjubsp (mgr.14720) 257 : cephadm [INF] Upgrade: Updating mds.foofs.smithi196.hhhkwt 2024-02-19T21:59:25.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:24.079096+0000 mgr.smithi131.cjubsp (mgr.14720) 258 : cephadm [INF] Deploying daemon mds.foofs.smithi196.hhhkwt on smithi196 2024-02-19T21:59:26.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:24.587734+0000 mgr.smithi131.cjubsp (mgr.14720) 259 : cluster [DBG] pgmap v195: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.7 KiB/s wr, 8 op/s 2024-02-19T21:59:26.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:25.864392+0000 mon.smithi131 (mon.0) 690 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-02-19T21:59:26.674 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:59:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:59:26.286+0000 7efc20509700 -1 bdev(0x555c21bfa800 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 4 2024-02-19T21:59:26.674 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 21:59:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T21:59:26.287+0000 7efc2150b700 -1 bdev(0x555c21bfa800 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 4 2024-02-19T21:59:26.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T21:59:26.263+0000 7fbb0cfd7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:59:26.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:24.587734+0000 mgr.smithi131.cjubsp (mgr.14720) 259 : cluster [DBG] pgmap v195: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.7 KiB/s wr, 8 op/s 2024-02-19T21:59:26.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:25.864392+0000 mon.smithi131 (mon.0) 690 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-02-19T21:59:27.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:26.264299+0000 mon.smithi131 (mon.0) 691 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-02-19T21:59:27.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:26.264334+0000 mon.smithi131 (mon.0) 692 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:59:27.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:26.270572+0000 mon.smithi131 (mon.0) 693 : cluster [INF] Standby daemon mds.foofs.smithi131.gemwnz assigned to filesystem foofs as rank 0 2024-02-19T21:59:27.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:26.270675+0000 mon.smithi131 (mon.0) 694 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-02-19T21:59:27.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:26.270696+0000 mon.smithi131 (mon.0) 695 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-19T21:59:27.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:26.270909+0000 mon.smithi131 (mon.0) 696 : cluster [DBG] fsmap foofs:0/1 1 up:standby, 1 failed 2024-02-19T21:59:27.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:26.276027+0000 mon.smithi131 (mon.0) 697 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi131.gemwnz=up:replay} 2024-02-19T21:59:27.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:26.513624+0000 mon.smithi131 (mon.0) 698 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:59:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:26.264299+0000 mon.smithi131 (mon.0) 691 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-02-19T21:59:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:26.264334+0000 mon.smithi131 (mon.0) 692 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-19T21:59:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:26.270572+0000 mon.smithi131 (mon.0) 693 : cluster [INF] Standby daemon mds.foofs.smithi131.gemwnz assigned to filesystem foofs as rank 0 2024-02-19T21:59:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:26.270675+0000 mon.smithi131 (mon.0) 694 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-02-19T21:59:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:26.270696+0000 mon.smithi131 (mon.0) 695 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-19T21:59:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:26.270909+0000 mon.smithi131 (mon.0) 696 : cluster [DBG] fsmap foofs:0/1 1 up:standby, 1 failed 2024-02-19T21:59:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:26.276027+0000 mon.smithi131 (mon.0) 697 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi131.gemwnz=up:replay} 2024-02-19T21:59:27.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:26.513624+0000 mon.smithi131 (mon.0) 698 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:59:27.955 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T21:59:28.020 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:59:28.535 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (12m) 6s ago 13m 21.0M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (4m) 6s ago 13m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (4m) 22s ago 12m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (12m) 6s ago 13m 34.6M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (12s) 6s ago 8m 15.2M - 16.2.14-548-g81bd20d6 823b921721f5 9ed7e8639d27 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (8m) 22s ago 8m 13.9M - 16.2.5 6933c2a0b7dd aae0bbe4d5da 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (5m) 6s ago 15m 446M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (4m) 22s ago 12m 389M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (4m) 6s ago 15m 49.2M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (4m) 22s ago 12m 36.5M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (5m) 6s ago 5m 81.9M - 3.5 6933c2a0b7dd 4a0377af6144 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (5m) 22s ago 5m 79.2M - 3.5 6933c2a0b7dd fc5b445f3f5e 2024-02-19T21:59:28.536 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (13m) 6s ago 13m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (12m) 22s ago 12m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (4m) 6s ago 12m 118M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (3m) 6s ago 11m 108M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (2m) 6s ago 11m 88.4M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (2m) 6s ago 11m 118M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (113s) 22s ago 10m 118M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (84s) 22s ago 10m 112M 2712M 16.2.14-548-g81bd20d6 823b921721f5 4c4b41297954 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (56s) 22s ago 10m 100M 2712M 16.2.14-548-g81bd20d6 823b921721f5 e9be2f442ef3 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (27s) 22s ago 9m 10.4M 2712M 16.2.14-548-g81bd20d6 823b921721f5 10d1c2418ed8 2024-02-19T21:59:28.537 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (5m) 6s ago 13m 63.1M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T21:59:28.661 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:26.588159+0000 mgr.smithi131.cjubsp (mgr.14720) 260 : cluster [DBG] pgmap v197: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 6.9 KiB/s wr, 6 op/s 2024-02-19T21:59:28.661 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:27.279225+0000 mon.smithi131 (mon.0) 699 : cluster [DBG] mds.? [v2:172.21.15.131:6834/158099597,v1:172.21.15.131:6835/158099597] up:reconnect 2024-02-19T21:59:28.661 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:27.279270+0000 mon.smithi131 (mon.0) 700 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi131.gemwnz=up:reconnect} 2024-02-19T21:59:28.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:26.588159+0000 mgr.smithi131.cjubsp (mgr.14720) 260 : cluster [DBG] pgmap v197: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 6.9 KiB/s wr, 6 op/s 2024-02-19T21:59:28.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:27.279225+0000 mon.smithi131 (mon.0) 699 : cluster [DBG] mds.? [v2:172.21.15.131:6834/158099597,v1:172.21.15.131:6835/158099597] up:reconnect 2024-02-19T21:59:28.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:27.279270+0000 mon.smithi131 (mon.0) 700 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi131.gemwnz=up:reconnect} 2024-02-19T21:59:28.875 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:59:28.905 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T21:59:28.905 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 8 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 1 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 13 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T21:59:28.906 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout: "osd", 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T21:59:29.169 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr" 2024-02-19T21:59:29.170 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T21:59:29.170 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "15/23 daemons upgraded", 2024-02-19T21:59:29.170 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading mds daemons", 2024-02-19T21:59:29.170 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T21:59:29.170 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T21:59:29.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:27.943796+0000 mgr.smithi131.cjubsp (mgr.14720) 261 : audit [DBG] from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:59:29.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:28.243077+0000 mgr.smithi131.cjubsp (mgr.14720) 262 : audit [DBG] from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:59:29.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:28.284809+0000 mon.smithi131 (mon.0) 701 : cluster [DBG] mds.? [v2:172.21.15.131:6834/158099597,v1:172.21.15.131:6835/158099597] up:rejoin 2024-02-19T21:59:29.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:28.284863+0000 mon.smithi131 (mon.0) 702 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi131.gemwnz=up:rejoin} 2024-02-19T21:59:29.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:28.286468+0000 mon.smithi131 (mon.0) 703 : cluster [INF] daemon mds.foofs.smithi131.gemwnz is now active in filesystem foofs as rank 0 2024-02-19T21:59:29.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:28.876871+0000 mon.smithi131 (mon.0) 704 : audit [DBG] from='client.? 172.21.15.131:0/3144669678' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:27.943796+0000 mgr.smithi131.cjubsp (mgr.14720) 261 : audit [DBG] from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:59:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:28.243077+0000 mgr.smithi131.cjubsp (mgr.14720) 262 : audit [DBG] from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:59:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:28.284809+0000 mon.smithi131 (mon.0) 701 : cluster [DBG] mds.? [v2:172.21.15.131:6834/158099597,v1:172.21.15.131:6835/158099597] up:rejoin 2024-02-19T21:59:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:28.284863+0000 mon.smithi131 (mon.0) 702 : cluster [DBG] fsmap foofs:1/1 {0=foofs.smithi131.gemwnz=up:rejoin} 2024-02-19T21:59:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:28.286468+0000 mon.smithi131 (mon.0) 703 : cluster [INF] daemon mds.foofs.smithi131.gemwnz is now active in filesystem foofs as rank 0 2024-02-19T21:59:29.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:28.876871+0000 mon.smithi131 (mon.0) 704 : audit [DBG] from='client.? 172.21.15.131:0/3144669678' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:28.527744+0000 mgr.smithi131.cjubsp (mgr.14720) 263 : audit [DBG] from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:59:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:28.588604+0000 mgr.smithi131.cjubsp (mgr.14720) 264 : cluster [DBG] pgmap v198: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 204 B/s wr, 3 op/s 2024-02-19T21:59:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:29.169918+0000 mgr.smithi131.cjubsp (mgr.14720) 265 : audit [DBG] from='client.34318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:59:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:29.283141+0000 mon.smithi131 (mon.0) 705 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-02-19T21:59:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:29.289578+0000 mon.smithi131 (mon.0) 706 : cluster [DBG] mds.? [v2:172.21.15.131:6834/158099597,v1:172.21.15.131:6835/158099597] up:active 2024-02-19T21:59:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:29.289633+0000 mon.smithi131 (mon.0) 707 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 2024-02-19T21:59:30.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:28.527744+0000 mgr.smithi131.cjubsp (mgr.14720) 263 : audit [DBG] from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:59:30.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:28.588604+0000 mgr.smithi131.cjubsp (mgr.14720) 264 : cluster [DBG] pgmap v198: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 MiB/s rd, 204 B/s wr, 3 op/s 2024-02-19T21:59:30.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:29.169918+0000 mgr.smithi131.cjubsp (mgr.14720) 265 : audit [DBG] from='client.34318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T21:59:30.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:29.283141+0000 mon.smithi131 (mon.0) 705 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-02-19T21:59:30.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:29.289578+0000 mon.smithi131 (mon.0) 706 : cluster [DBG] mds.? [v2:172.21.15.131:6834/158099597,v1:172.21.15.131:6835/158099597] up:active 2024-02-19T21:59:30.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:29.289633+0000 mon.smithi131 (mon.0) 707 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 2024-02-19T21:59:31.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:30.589428+0000 mgr.smithi131.cjubsp (mgr.14720) 266 : cluster [DBG] pgmap v199: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 204 B/s wr, 5 op/s 2024-02-19T21:59:31.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:30.589428+0000 mgr.smithi131.cjubsp (mgr.14720) 266 : cluster [DBG] pgmap v199: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 204 B/s wr, 5 op/s 2024-02-19T21:59:33.063 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:59:33.804 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:32.590107+0000 mgr.smithi131.cjubsp (mgr.14720) 267 : cluster [DBG] pgmap v200: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 204 B/s wr, 9 op/s 2024-02-19T21:59:34.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:32.590107+0000 mgr.smithi131.cjubsp (mgr.14720) 267 : cluster [DBG] pgmap v200: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 204 B/s wr, 9 op/s 2024-02-19T21:59:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:34.590748+0000 mgr.smithi131.cjubsp (mgr.14720) 268 : cluster [DBG] pgmap v201: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 6.9 KiB/s wr, 13 op/s 2024-02-19T21:59:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:35.088111+0000 mon.smithi131 (mon.0) 708 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:35.088978+0000 mon.smithi131 (mon.0) 709 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:36.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:35.645531+0000 mon.smithi131 (mon.0) 710 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-02-19T21:59:36.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:35.645592+0000 mon.smithi131 (mon.0) 711 : cluster [INF] Cluster is now healthy 2024-02-19T21:59:36.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:35.649296+0000 mon.smithi131 (mon.0) 712 : cluster [DBG] mds.? [v2:172.21.15.196:6832/2136736134,v1:172.21.15.196:6833/2136736134] up:boot 2024-02-19T21:59:36.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:35.649356+0000 mon.smithi131 (mon.0) 713 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:59:36.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:35.649518+0000 mon.smithi131 (mon.0) 714 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:59:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:34.590748+0000 mgr.smithi131.cjubsp (mgr.14720) 268 : cluster [DBG] pgmap v201: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 6.9 KiB/s wr, 13 op/s 2024-02-19T21:59:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:35.088111+0000 mon.smithi131 (mon.0) 708 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:35.088978+0000 mon.smithi131 (mon.0) 709 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:35.645531+0000 mon.smithi131 (mon.0) 710 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-02-19T21:59:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:35.645592+0000 mon.smithi131 (mon.0) 711 : cluster [INF] Cluster is now healthy 2024-02-19T21:59:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:35.649296+0000 mon.smithi131 (mon.0) 712 : cluster [DBG] mds.? [v2:172.21.15.196:6832/2136736134,v1:172.21.15.196:6833/2136736134] up:boot 2024-02-19T21:59:36.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:35.649356+0000 mon.smithi131 (mon.0) 713 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi131.gemwnz=up:active} 1 up:standby 2024-02-19T21:59:36.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:35.649518+0000 mon.smithi131 (mon.0) 714 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:59:38.106 INFO:teuthology.orchestra.run.smithi131.stderr:mount.nfs: mounting smithi131:/fake failed, reason given by server: No such file or directory 2024-02-19T21:59:38.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:36.591362+0000 mgr.smithi131.cjubsp (mgr.14720) 269 : cluster [DBG] pgmap v202: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19 MiB/s rd, 6.4 KiB/s wr, 12 op/s 2024-02-19T21:59:38.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.077898+0000 mon.smithi131 (mon.0) 715 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:38.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.079005+0000 mon.smithi131 (mon.0) 716 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:38.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.080008+0000 mon.smithi131 (mon.0) 717 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:59:38.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.087223+0000 mon.smithi131 (mon.0) 718 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:36.591362+0000 mgr.smithi131.cjubsp (mgr.14720) 269 : cluster [DBG] pgmap v202: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 19 MiB/s rd, 6.4 KiB/s wr, 12 op/s 2024-02-19T21:59:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.077898+0000 mon.smithi131 (mon.0) 715 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.079005+0000 mon.smithi131 (mon.0) 716 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:38.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.080008+0000 mon.smithi131 (mon.0) 717 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:59:38.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.087223+0000 mon.smithi131 (mon.0) 718 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.095015+0000 mon.smithi131 (mon.0) 719 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.098441+0000 mon.smithi131 (mon.0) 720 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.100719+0000 mon.smithi131 (mon.0) 721 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.101986+0000 mon.smithi131 (mon.0) 722 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.103244+0000 mon.smithi131 (mon.0) 723 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.105574+0000 mon.smithi131 (mon.0) 724 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.108177+0000 mon.smithi131 (mon.0) 725 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:38.108971+0000 mgr.smithi131.cjubsp (mgr.14720) 270 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.113150+0000 mon.smithi131 (mon.0) 726 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.113894+0000 mon.smithi131 (mon.0) 727 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.117457+0000 mon.smithi131 (mon.0) 728 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi131.gemwnz"}]': finished 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.118086+0000 mon.smithi131 (mon.0) 729 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.122695+0000 mon.smithi131 (mon.0) 730 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi196.hhhkwt"}]': finished 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.123697+0000 mon.smithi131 (mon.0) 731 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:38.124297+0000 mgr.smithi131.cjubsp (mgr.14720) 271 : cephadm [INF] Upgrade: Setting container_image for all rgw 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.127387+0000 mon.smithi131 (mon.0) 732 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.128294+0000 mon.smithi131 (mon.0) 733 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:38.128842+0000 mgr.smithi131.cjubsp (mgr.14720) 272 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-02-19T21:59:39.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.132980+0000 mon.smithi131 (mon.0) 734 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.133855+0000 mon.smithi131 (mon.0) 735 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.134799+0000 mon.smithi131 (mon.0) 736 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:38.135317+0000 mgr.smithi131.cjubsp (mgr.14720) 273 : cephadm [INF] Upgrade: Setting container_image for all iscsi 2024-02-19T21:59:39.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:38.138360+0000 mon.smithi131 (mon.0) 737 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.095015+0000 mon.smithi131 (mon.0) 719 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.098441+0000 mon.smithi131 (mon.0) 720 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.100719+0000 mon.smithi131 (mon.0) 721 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.101986+0000 mon.smithi131 (mon.0) 722 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.103244+0000 mon.smithi131 (mon.0) 723 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.105574+0000 mon.smithi131 (mon.0) 724 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.108177+0000 mon.smithi131 (mon.0) 725 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:38.108971+0000 mgr.smithi131.cjubsp (mgr.14720) 270 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.113150+0000 mon.smithi131 (mon.0) 726 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.113894+0000 mon.smithi131 (mon.0) 727 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi131.gemwnz"}]: dispatch 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.117457+0000 mon.smithi131 (mon.0) 728 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi131.gemwnz"}]': finished 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.118086+0000 mon.smithi131 (mon.0) 729 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi196.hhhkwt"}]: dispatch 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.122695+0000 mon.smithi131 (mon.0) 730 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi196.hhhkwt"}]': finished 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.123697+0000 mon.smithi131 (mon.0) 731 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:38.124297+0000 mgr.smithi131.cjubsp (mgr.14720) 271 : cephadm [INF] Upgrade: Setting container_image for all rgw 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.127387+0000 mon.smithi131 (mon.0) 732 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.128294+0000 mon.smithi131 (mon.0) 733 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:38.128842+0000 mgr.smithi131.cjubsp (mgr.14720) 272 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-02-19T21:59:39.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.132980+0000 mon.smithi131 (mon.0) 734 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:39.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.133855+0000 mon.smithi131 (mon.0) 735 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.134799+0000 mon.smithi131 (mon.0) 736 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:39.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:38.135317+0000 mgr.smithi131.cjubsp (mgr.14720) 273 : cephadm [INF] Upgrade: Setting container_image for all iscsi 2024-02-19T21:59:39.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:38.138360+0000 mon.smithi131 (mon.0) 737 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:40.410 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:38.592150+0000 mgr.smithi131.cjubsp (mgr.14720) 274 : cluster [DBG] pgmap v203: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-02-19T21:59:40.410 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.569986+0000 mon.smithi131 (mon.0) 738 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:40.410 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.570844+0000 mon.smithi131 (mon.0) 739 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-19T21:59:40.411 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.571492+0000 mon.smithi131 (mon.0) 740 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-19T21:59:40.411 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.576365+0000 mon.smithi131 (mon.0) 741 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-19T21:59:40.411 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.577297+0000 mon.smithi131 (mon.0) 742 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:40.411 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.615994+0000 mon.smithi131 (mon.0) 743 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:59:40.411 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.621230+0000 mon.smithi131 (mon.0) 744 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:59:40.411 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.666504+0000 mon.smithi131 (mon.0) 745 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:59:40.411 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:39.672358+0000 mon.smithi131 (mon.0) 746 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:38.592150+0000 mgr.smithi131.cjubsp (mgr.14720) 274 : cluster [DBG] pgmap v203: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-02-19T21:59:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.569986+0000 mon.smithi131 (mon.0) 738 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.570844+0000 mon.smithi131 (mon.0) 739 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-19T21:59:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.571492+0000 mon.smithi131 (mon.0) 740 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-19T21:59:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.576365+0000 mon.smithi131 (mon.0) 741 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-19T21:59:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.577297+0000 mon.smithi131 (mon.0) 742 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.615994+0000 mon.smithi131 (mon.0) 743 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:59:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.621230+0000 mon.smithi131 (mon.0) 744 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:59:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.666504+0000 mon.smithi131 (mon.0) 745 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.1.smithi131.qccypf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:59:40.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:39.672358+0000 mon.smithi131 (mon.0) 746 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:39.565698+0000 mgr.smithi131.cjubsp (mgr.14720) 275 : cephadm [INF] Upgrade: Updating nfs.foo.0.1.smithi131.qccypf (1/2) 2024-02-19T21:59:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:39.570589+0000 mgr.smithi131.cjubsp (mgr.14720) 276 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi131.qccypf 2024-02-19T21:59:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:39.571324+0000 mgr.smithi131.cjubsp (mgr.14720) 277 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-19T21:59:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:39.666067+0000 mgr.smithi131.cjubsp (mgr.14720) 278 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:59:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:39.666158+0000 mgr.smithi131.cjubsp (mgr.14720) 279 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi131.qccypf-rgw 2024-02-19T21:59:41.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:39.673069+0000 mgr.smithi131.cjubsp (mgr.14720) 280 : cephadm [INF] Deploying daemon nfs.foo.0.1.smithi131.qccypf on smithi131 2024-02-19T21:59:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:39.565698+0000 mgr.smithi131.cjubsp (mgr.14720) 275 : cephadm [INF] Upgrade: Updating nfs.foo.0.1.smithi131.qccypf (1/2) 2024-02-19T21:59:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:39.570589+0000 mgr.smithi131.cjubsp (mgr.14720) 276 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi131.qccypf 2024-02-19T21:59:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:39.571324+0000 mgr.smithi131.cjubsp (mgr.14720) 277 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-19T21:59:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:39.666067+0000 mgr.smithi131.cjubsp (mgr.14720) 278 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:59:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:39.666158+0000 mgr.smithi131.cjubsp (mgr.14720) 279 : cephadm [INF] Creating key for client.nfs.foo.0.1.smithi131.qccypf-rgw 2024-02-19T21:59:41.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:39.673069+0000 mgr.smithi131.cjubsp (mgr.14720) 280 : cephadm [INF] Deploying daemon nfs.foo.0.1.smithi131.qccypf on smithi131 2024-02-19T21:59:42.348 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:40.592916+0000 mgr.smithi131.cjubsp (mgr.14720) 281 : cluster [DBG] pgmap v204: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.7 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-02-19T21:59:42.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:40.592916+0000 mgr.smithi131.cjubsp (mgr.14720) 281 : cluster [DBG] pgmap v204: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.7 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-02-19T21:59:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:42.593388+0000 mgr.smithi131.cjubsp (mgr.14720) 282 : cluster [DBG] pgmap v205: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-02-19T21:59:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:43.184368+0000 mon.smithi131 (mon.0) 747 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:44.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:42.593388+0000 mgr.smithi131.cjubsp (mgr.14720) 282 : cluster [DBG] pgmap v205: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-02-19T21:59:44.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:43.184368+0000 mon.smithi131 (mon.0) 747 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:46.086 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:44.593968+0000 mgr.smithi131.cjubsp (mgr.14720) 283 : cluster [DBG] pgmap v206: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 8.1 KiB/s wr, 7 op/s 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:44.826884+0000 mgr.smithi131.cjubsp (mgr.14720) 284 : cephadm [INF] Upgrade: Updating nfs.foo.1.1.smithi196.rqnuju (2/2) 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.831622+0000 mon.smithi131 (mon.0) 748 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:44.832399+0000 mgr.smithi131.cjubsp (mgr.14720) 285 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi196.rqnuju 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.832695+0000 mon.smithi131 (mon.0) 749 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:44.833338+0000 mgr.smithi131.cjubsp (mgr.14720) 286 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.833603+0000 mon.smithi131 (mon.0) 750 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.836765+0000 mon.smithi131 (mon.0) 751 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.837986+0000 mon.smithi131 (mon.0) 752 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.882577+0000 mon.smithi131 (mon.0) 753 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.886273+0000 mon.smithi131 (mon.0) 754 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:44.933791+0000 mgr.smithi131.cjubsp (mgr.14720) 287 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:59:46.087 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:44.933935+0000 mgr.smithi131.cjubsp (mgr.14720) 288 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi196.rqnuju-rgw 2024-02-19T21:59:46.088 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.934284+0000 mon.smithi131 (mon.0) 755 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:59:46.088 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:44.935373+0000 mon.smithi131 (mon.0) 756 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:46.088 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:44.936084+0000 mgr.smithi131.cjubsp (mgr.14720) 289 : cephadm [INF] Deploying daemon nfs.foo.1.1.smithi196.rqnuju on smithi196 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:44.593968+0000 mgr.smithi131.cjubsp (mgr.14720) 283 : cluster [DBG] pgmap v206: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 8.1 KiB/s wr, 7 op/s 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:44.826884+0000 mgr.smithi131.cjubsp (mgr.14720) 284 : cephadm [INF] Upgrade: Updating nfs.foo.1.1.smithi196.rqnuju (2/2) 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.831622+0000 mon.smithi131 (mon.0) 748 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:44.832399+0000 mgr.smithi131.cjubsp (mgr.14720) 285 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi196.rqnuju 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.832695+0000 mon.smithi131 (mon.0) 749 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:44.833338+0000 mgr.smithi131.cjubsp (mgr.14720) 286 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.833603+0000 mon.smithi131 (mon.0) 750 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.836765+0000 mon.smithi131 (mon.0) 751 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.837986+0000 mon.smithi131 (mon.0) 752 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:46.140 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.882577+0000 mon.smithi131 (mon.0) 753 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-19T21:59:46.141 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.886273+0000 mon.smithi131 (mon.0) 754 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-19T21:59:46.141 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:44.933791+0000 mgr.smithi131.cjubsp (mgr.14720) 287 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-02-19T21:59:46.141 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:44.933935+0000 mgr.smithi131.cjubsp (mgr.14720) 288 : cephadm [INF] Creating key for client.nfs.foo.1.1.smithi196.rqnuju-rgw 2024-02-19T21:59:46.141 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.934284+0000 mon.smithi131 (mon.0) 755 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.1.smithi196.rqnuju-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-19T21:59:46.141 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:44.935373+0000 mon.smithi131 (mon.0) 756 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:46.141 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:44.936084+0000 mgr.smithi131.cjubsp (mgr.14720) 289 : cephadm [INF] Deploying daemon nfs.foo.1.1.smithi196.rqnuju on smithi196 2024-02-19T21:59:46.175 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-02-19T21:59:46.207 INFO:teuthology.orchestra.run.smithi131.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-02-19T21:59:46.207 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:59:46.208 INFO:teuthology.orchestra.run.smithi131.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-02-19T21:59:46.208 INFO:teuthology.orchestra.run.smithi131.stdout:3 of 5 processes prepared for launch 0 sec 2024-02-19T21:59:46.308 INFO:teuthology.orchestra.run.smithi131.stdout:5 of 5 processes prepared for launch 0 sec 2024-02-19T21:59:46.308 INFO:teuthology.orchestra.run.smithi131.stdout:releasing clients 2024-02-19T21:59:48.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:46.594548+0000 mgr.smithi131.cjubsp (mgr.14720) 290 : cluster [DBG] pgmap v207: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.7 KiB/s wr, 4 op/s 2024-02-19T21:59:48.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:46.594548+0000 mgr.smithi131.cjubsp (mgr.14720) 290 : cluster [DBG] pgmap v207: 129 pgs: 129 active+clean; 304 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.7 KiB/s wr, 4 op/s 2024-02-19T21:59:48.309 INFO:teuthology.orchestra.run.smithi131.stdout: 5 179 0.00 MB/sec execute 1 sec latency 1301.254 ms 2024-02-19T21:59:49.309 INFO:teuthology.orchestra.run.smithi131.stdout: 5 179 0.00 MB/sec execute 2 sec latency 2301.296 ms 2024-02-19T21:59:49.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:48.595045+0000 mgr.smithi131.cjubsp (mgr.14720) 291 : cluster [DBG] pgmap v208: 129 pgs: 129 active+clean; 313 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 852 KiB/s wr, 37 op/s 2024-02-19T21:59:49.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:48.668040+0000 mon.smithi131 (mon.0) 757 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:49.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:48.668802+0000 mon.smithi131 (mon.0) 758 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:48.595045+0000 mgr.smithi131.cjubsp (mgr.14720) 291 : cluster [DBG] pgmap v208: 129 pgs: 129 active+clean; 313 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 852 KiB/s wr, 37 op/s 2024-02-19T21:59:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:48.668040+0000 mon.smithi131 (mon.0) 757 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:48.668802+0000 mon.smithi131 (mon.0) 758 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:50.309 INFO:teuthology.orchestra.run.smithi131.stdout: 5 179 0.00 MB/sec execute 3 sec latency 3301.321 ms 2024-02-19T21:59:51.309 INFO:teuthology.orchestra.run.smithi131.stdout: 5 179 0.00 MB/sec execute 4 sec latency 4301.393 ms 2024-02-19T21:59:51.896 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:50.595578+0000 mgr.smithi131.cjubsp (mgr.14720) 292 : cluster [DBG] pgmap v209: 129 pgs: 129 active+clean; 280 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 2.4 MiB/s wr, 63 op/s 2024-02-19T21:59:51.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:50.595578+0000 mgr.smithi131.cjubsp (mgr.14720) 292 : cluster [DBG] pgmap v209: 129 pgs: 129 active+clean; 280 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 2.4 MiB/s wr, 63 op/s 2024-02-19T21:59:52.309 INFO:teuthology.orchestra.run.smithi131.stdout: 5 cleanup 5 sec 2024-02-19T21:59:53.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:51.848634+0000 mon.smithi131 (mon.0) 759 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:53.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:51.848634+0000 mon.smithi131 (mon.0) 759 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:53.309 INFO:teuthology.orchestra.run.smithi131.stdout: 5 cleanup 6 sec 2024-02-19T21:59:53.582 INFO:teuthology.orchestra.run.smithi131.stdout: 0 cleanup 6 sec 2024-02-19T21:59:53.582 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:59:53.582 INFO:teuthology.orchestra.run.smithi131.stdout: Operation Count AvgLat MaxLat 2024-02-19T21:59:53.582 INFO:teuthology.orchestra.run.smithi131.stdout: ---------------------------------------- 2024-02-19T21:59:53.582 INFO:teuthology.orchestra.run.smithi131.stdout: NTCreateX 5 6507.756 6519.827 2024-02-19T21:59:53.582 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T21:59:53.582 INFO:teuthology.orchestra.run.smithi131.stdout:Throughput 0 MB/sec 5 clients 5 procs max_latency=4301.393 ms 2024-02-19T21:59:53.584 DEBUG:teuthology.parallel:result is None 2024-02-19T21:59:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:52.596132+0000 mgr.smithi131.cjubsp (mgr.14720) 293 : cluster [DBG] pgmap v210: 129 pgs: 129 active+clean; 267 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 3.1 MiB/s wr, 75 op/s 2024-02-19T21:59:54.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:52.596132+0000 mgr.smithi131.cjubsp (mgr.14720) 293 : cluster [DBG] pgmap v210: 129 pgs: 129 active+clean; 267 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 3.1 MiB/s wr, 75 op/s 2024-02-19T21:59:55.162 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.148034+0000 mon.smithi131 (mon.0) 760 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:55.162 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.148865+0000 mon.smithi131 (mon.0) 761 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.149485+0000 mon.smithi131 (mon.0) 762 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.153404+0000 mon.smithi131 (mon.0) 763 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.158452+0000 mon.smithi131 (mon.0) 764 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.161303+0000 mon.smithi131 (mon.0) 765 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.163399+0000 mon.smithi131 (mon.0) 766 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.164509+0000 mon.smithi131 (mon.0) 767 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.165614+0000 mon.smithi131 (mon.0) 768 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.167097+0000 mon.smithi131 (mon.0) 769 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.168671+0000 mon.smithi131 (mon.0) 770 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.169588+0000 mon.smithi131 (mon.0) 771 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.170478+0000 mon.smithi131 (mon.0) 772 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.171377+0000 mon.smithi131 (mon.0) 773 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.172319+0000 mon.smithi131 (mon.0) 774 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.163 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.173446+0000 mon.smithi131 (mon.0) 775 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.164 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:54.173974+0000 mgr.smithi131.cjubsp (mgr.14720) 294 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-02-19T21:59:55.164 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.177471+0000 mon.smithi131 (mon.0) 776 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:55.164 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.178172+0000 mon.smithi131 (mon.0) 777 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.1.smithi131.qccypf"}]: dispatch 2024-02-19T21:59:55.164 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.181217+0000 mon.smithi131 (mon.0) 778 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.1.smithi131.qccypf"}]': finished 2024-02-19T21:59:55.164 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.181720+0000 mon.smithi131 (mon.0) 779 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.1.smithi196.rqnuju"}]: dispatch 2024-02-19T21:59:55.164 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:54.184685+0000 mon.smithi131 (mon.0) 780 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.1.smithi196.rqnuju"}]': finished 2024-02-19T21:59:55.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.148034+0000 mon.smithi131 (mon.0) 760 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.148865+0000 mon.smithi131 (mon.0) 761 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.149485+0000 mon.smithi131 (mon.0) 762 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.153404+0000 mon.smithi131 (mon.0) 763 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.158452+0000 mon.smithi131 (mon.0) 764 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.161303+0000 mon.smithi131 (mon.0) 765 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.163399+0000 mon.smithi131 (mon.0) 766 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.164509+0000 mon.smithi131 (mon.0) 767 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.165614+0000 mon.smithi131 (mon.0) 768 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.167097+0000 mon.smithi131 (mon.0) 769 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.168671+0000 mon.smithi131 (mon.0) 770 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.169588+0000 mon.smithi131 (mon.0) 771 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.170478+0000 mon.smithi131 (mon.0) 772 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.171377+0000 mon.smithi131 (mon.0) 773 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.172319+0000 mon.smithi131 (mon.0) 774 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.173446+0000 mon.smithi131 (mon.0) 775 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:54.173974+0000 mgr.smithi131.cjubsp (mgr.14720) 294 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.177471+0000 mon.smithi131 (mon.0) 776 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.178172+0000 mon.smithi131 (mon.0) 777 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.1.smithi131.qccypf"}]: dispatch 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.181217+0000 mon.smithi131 (mon.0) 778 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.1.smithi131.qccypf"}]': finished 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.181720+0000 mon.smithi131 (mon.0) 779 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.1.smithi196.rqnuju"}]: dispatch 2024-02-19T21:59:55.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:54.184685+0000 mon.smithi131 (mon.0) 780 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.1.smithi196.rqnuju"}]': finished 2024-02-19T21:59:56.477 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:54.596603+0000 mgr.smithi131.cjubsp (mgr.14720) 295 : cluster [DBG] pgmap v211: 129 pgs: 129 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 3.1 MiB/s wr, 83 op/s 2024-02-19T21:59:56.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:54.596603+0000 mgr.smithi131.cjubsp (mgr.14720) 295 : cluster [DBG] pgmap v211: 129 pgs: 129 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 3.1 MiB/s wr, 83 op/s 2024-02-19T21:59:57.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:55.495023+0000 mgr.smithi131.cjubsp (mgr.14720) 296 : cephadm [INF] Upgrade: Updating node-exporter.smithi131 (1/2) 2024-02-19T21:59:57.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T21:59:55.495176+0000 mgr.smithi131.cjubsp (mgr.14720) 297 : cephadm [INF] Deploying daemon node-exporter.smithi131 on smithi131 2024-02-19T21:59:57.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:55.495023+0000 mgr.smithi131.cjubsp (mgr.14720) 296 : cephadm [INF] Upgrade: Updating node-exporter.smithi131 (1/2) 2024-02-19T21:59:57.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T21:59:55.495176+0000 mgr.smithi131.cjubsp (mgr.14720) 297 : cephadm [INF] Deploying daemon node-exporter.smithi131 on smithi131 2024-02-19T21:59:58.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 21:59:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:56.597128+0000 mgr.smithi131.cjubsp (mgr.14720) 298 : cluster [DBG] pgmap v212: 129 pgs: 129 active+clean; 235 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 KiB/s rd, 3.1 MiB/s wr, 81 op/s 2024-02-19T21:59:58.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 21:59:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:56.597128+0000 mgr.smithi131.cjubsp (mgr.14720) 298 : cluster [DBG] pgmap v212: 129 pgs: 129 active+clean; 235 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 KiB/s rd, 3.1 MiB/s wr, 81 op/s 2024-02-19T21:59:59.481 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (12m) 5s ago 14m 21.0M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (4m) 5s ago 14m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (4m) 8s ago 13m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (12m) 5s ago 14m 34.7M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (43s) 5s ago 8m 36.9M - 16.2.14-548-g81bd20d6 823b921721f5 9ed7e8639d27 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (25s) 8s ago 8m 14.5M - 16.2.14-548-g81bd20d6 823b921721f5 b7eb2031d093 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (5m) 5s ago 15m 447M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (5m) 8s ago 13m 389M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (5m) 5s ago 16m 51.0M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T22:00:00.069 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (5m) 8s ago 13m 40.7M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (17s) 5s ago 6m 122M - 3.5 823b921721f5 b99130858f09 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (11s) 8s ago 6m 16.4M - 3.5 823b921721f5 faf5deba11ee 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (14m) 5s ago 13m 17.2M - 0.18.1 e5a616e4b9cf a817d814dca8 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (13m) 8s ago 13m 17.1M - 0.18.1 e5a616e4b9cf 4aff6479f4b7 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (4m) 5s ago 12m 123M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (4m) 5s ago 12m 114M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (3m) 5s ago 12m 93.4M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (2m) 5s ago 11m 121M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (2m) 8s ago 11m 128M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (116s) 8s ago 11m 120M 2712M 16.2.14-548-g81bd20d6 823b921721f5 4c4b41297954 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (87s) 8s ago 10m 113M 2712M 16.2.14-548-g81bd20d6 823b921721f5 e9be2f442ef3 2024-02-19T22:00:00.070 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (58s) 8s ago 10m 138M 2712M 16.2.14-548-g81bd20d6 823b921721f5 10d1c2418ed8 2024-02-19T22:00:00.071 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (5m) 5s ago 13m 66.3M - 2.18.1 de242295e225 6a4404f25fab 2024-02-19T22:00:00.425 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T22:00:00.425 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T22:00:00.425 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 8 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 14 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T22:00:00.426 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T22:00:00.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T21:59:58.597623+0000 mgr.smithi131.cjubsp (mgr.14720) 299 : cluster [DBG] pgmap v213: 129 pgs: 129 active+clean; 229 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 40 KiB/s rd, 3.1 MiB/s wr, 83 op/s 2024-02-19T22:00:00.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:00.000101+0000 mon.smithi131 (mon.0) 781 : cluster [INF] overall HEALTH_OK 2024-02-19T22:00:00.631 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T21:59:58.597623+0000 mgr.smithi131.cjubsp (mgr.14720) 299 : cluster [DBG] pgmap v213: 129 pgs: 129 active+clean; 229 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 40 KiB/s rd, 3.1 MiB/s wr, 83 op/s 2024-02-19T22:00:00.632 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:00.000101+0000 mon.smithi131 (mon.0) 781 : cluster [INF] overall HEALTH_OK 2024-02-19T22:00:00.716 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T22:00:00.716 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T22:00:00.716 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T22:00:00.716 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T22:00:00.716 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T22:00:00.716 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr", 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: "osd", 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: "nfs", 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: "mds" 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "18/23 daemons upgraded", 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading node-exporter daemons", 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T22:00:00.717 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T22:00:01.381 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:59.469256+0000 mgr.smithi131.cjubsp (mgr.14720) 300 : audit [DBG] from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:01.381 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T21:59:59.765684+0000 mgr.smithi131.cjubsp (mgr.14720) 301 : audit [DBG] from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:01.381 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:00.060639+0000 mgr.smithi131.cjubsp (mgr.14720) 302 : audit [DBG] from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:01.381 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:00.297259+0000 mon.smithi131 (mon.0) 782 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:01.381 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:00.425819+0000 mon.smithi131 (mon.0) 783 : audit [DBG] from='client.? 172.21.15.131:0/1812784628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:01.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:59.469256+0000 mgr.smithi131.cjubsp (mgr.14720) 300 : audit [DBG] from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:01.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T21:59:59.765684+0000 mgr.smithi131.cjubsp (mgr.14720) 301 : audit [DBG] from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:01.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:00.060639+0000 mgr.smithi131.cjubsp (mgr.14720) 302 : audit [DBG] from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:01.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:00.297259+0000 mon.smithi131 (mon.0) 782 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:01.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:00.425819+0000 mon.smithi131 (mon.0) 783 : audit [DBG] from='client.? 172.21.15.131:0/1812784628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:00.598129+0000 mgr.smithi131.cjubsp (mgr.14720) 303 : cluster [DBG] pgmap v214: 129 pgs: 129 active+clean; 211 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 2.3 MiB/s wr, 53 op/s 2024-02-19T22:00:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:00.717037+0000 mgr.smithi131.cjubsp (mgr.14720) 304 : audit [DBG] from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:02.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:00.598129+0000 mgr.smithi131.cjubsp (mgr.14720) 303 : cluster [DBG] pgmap v214: 129 pgs: 129 active+clean; 211 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 2.3 MiB/s wr, 53 op/s 2024-02-19T22:00:02.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:00.717037+0000 mgr.smithi131.cjubsp (mgr.14720) 304 : audit [DBG] from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:00:01.798182+0000 mgr.smithi131.cjubsp (mgr.14720) 305 : cephadm [INF] Upgrade: Updating node-exporter.smithi196 (2/2) 2024-02-19T22:00:03.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:00:01.798369+0000 mgr.smithi131.cjubsp (mgr.14720) 306 : cephadm [INF] Deploying daemon node-exporter.smithi196 on smithi196 2024-02-19T22:00:03.567 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:00:01.798182+0000 mgr.smithi131.cjubsp (mgr.14720) 305 : cephadm [INF] Upgrade: Updating node-exporter.smithi196 (2/2) 2024-02-19T22:00:03.568 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:00:01.798369+0000 mgr.smithi131.cjubsp (mgr.14720) 306 : cephadm [INF] Deploying daemon node-exporter.smithi196 on smithi196 2024-02-19T22:00:04.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:02.598854+0000 mgr.smithi131.cjubsp (mgr.14720) 307 : cluster [DBG] pgmap v215: 129 pgs: 129 active+clean; 207 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 768 KiB/s wr, 28 op/s 2024-02-19T22:00:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:02.598854+0000 mgr.smithi131.cjubsp (mgr.14720) 307 : cluster [DBG] pgmap v215: 129 pgs: 129 active+clean; 207 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 768 KiB/s wr, 28 op/s 2024-02-19T22:00:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:04.599584+0000 mgr.smithi131.cjubsp (mgr.14720) 308 : cluster [DBG] pgmap v216: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1023 B/s rd, 61 KiB/s wr, 18 op/s 2024-02-19T22:00:06.530 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:04.599584+0000 mgr.smithi131.cjubsp (mgr.14720) 308 : cluster [DBG] pgmap v216: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1023 B/s rd, 61 KiB/s wr, 18 op/s 2024-02-19T22:00:07.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:06.594817+0000 mon.smithi131 (mon.0) 784 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:07.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:06.595700+0000 mon.smithi131 (mon.0) 785 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:07.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:06.600219+0000 mgr.smithi131.cjubsp (mgr.14720) 309 : cluster [DBG] pgmap v217: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 938 B/s rd, 44 KiB/s wr, 10 op/s 2024-02-19T22:00:08.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:06.594817+0000 mon.smithi131 (mon.0) 784 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:08.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:06.595700+0000 mon.smithi131 (mon.0) 785 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:08.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:06.600219+0000 mgr.smithi131.cjubsp (mgr.14720) 309 : cluster [DBG] pgmap v217: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 938 B/s rd, 44 KiB/s wr, 10 op/s 2024-02-19T22:00:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:08.600958+0000 mgr.smithi131.cjubsp (mgr.14720) 310 : cluster [DBG] pgmap v218: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 938 B/s rd, 43 KiB/s wr, 9 op/s 2024-02-19T22:00:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:09.514037+0000 mon.smithi131 (mon.0) 786 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:10.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:08.600958+0000 mgr.smithi131.cjubsp (mgr.14720) 310 : cluster [DBG] pgmap v218: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 938 B/s rd, 43 KiB/s wr, 9 op/s 2024-02-19T22:00:10.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:09.514037+0000 mon.smithi131 (mon.0) 786 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:10.601546+0000 mgr.smithi131.cjubsp (mgr.14720) 311 : cluster [DBG] pgmap v219: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 426 B/s rd, 4.7 KiB/s wr, 5 op/s 2024-02-19T22:00:12.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:10.601546+0000 mgr.smithi131.cjubsp (mgr.14720) 311 : cluster [DBG] pgmap v219: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 426 B/s rd, 4.7 KiB/s wr, 5 op/s 2024-02-19T22:00:13.519 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.435068+0000 mon.smithi131 (mon.0) 787 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:13.519 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.435809+0000 mon.smithi131 (mon.0) 788 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.436354+0000 mon.smithi131 (mon.0) 789 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.439744+0000 mon.smithi131 (mon.0) 790 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.445347+0000 mon.smithi131 (mon.0) 791 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.448047+0000 mon.smithi131 (mon.0) 792 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.449829+0000 mon.smithi131 (mon.0) 793 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.450877+0000 mon.smithi131 (mon.0) 794 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.451898+0000 mon.smithi131 (mon.0) 795 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.453346+0000 mon.smithi131 (mon.0) 796 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.454848+0000 mon.smithi131 (mon.0) 797 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.455709+0000 mon.smithi131 (mon.0) 798 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.456492+0000 mon.smithi131 (mon.0) 799 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.520 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.457317+0000 mon.smithi131 (mon.0) 800 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.521 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.458229+0000 mon.smithi131 (mon.0) 801 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.521 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.459334+0000 mon.smithi131 (mon.0) 802 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.521 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:12.460560+0000 mon.smithi131 (mon.0) 803 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.521 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:13 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:12.602165+0000 mgr.smithi131.cjubsp (mgr.14720) 312 : cluster [DBG] pgmap v220: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 426 B/s rd, 4.7 KiB/s wr, 3 op/s 2024-02-19T22:00:13.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.435068+0000 mon.smithi131 (mon.0) 787 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:13.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.435809+0000 mon.smithi131 (mon.0) 788 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:00:13.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.436354+0000 mon.smithi131 (mon.0) 789 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:00:13.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.439744+0000 mon.smithi131 (mon.0) 790 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.445347+0000 mon.smithi131 (mon.0) 791 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.448047+0000 mon.smithi131 (mon.0) 792 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.449829+0000 mon.smithi131 (mon.0) 793 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.450877+0000 mon.smithi131 (mon.0) 794 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.451898+0000 mon.smithi131 (mon.0) 795 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.453346+0000 mon.smithi131 (mon.0) 796 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.454848+0000 mon.smithi131 (mon.0) 797 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.455709+0000 mon.smithi131 (mon.0) 798 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.456492+0000 mon.smithi131 (mon.0) 799 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.457317+0000 mon.smithi131 (mon.0) 800 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.458229+0000 mon.smithi131 (mon.0) 801 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.459334+0000 mon.smithi131 (mon.0) 802 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:12.460560+0000 mon.smithi131 (mon.0) 803 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:13.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:12.602165+0000 mgr.smithi131.cjubsp (mgr.14720) 312 : cluster [DBG] pgmap v220: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 426 B/s rd, 4.7 KiB/s wr, 3 op/s 2024-02-19T22:00:14.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:00:13.984106+0000 mgr.smithi131.cjubsp (mgr.14720) 313 : cephadm [INF] Upgrade: Updating prometheus.smithi131 2024-02-19T22:00:14.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:00:13.992111+0000 mgr.smithi131.cjubsp (mgr.14720) 314 : cephadm [INF] Deploying daemon prometheus.smithi131 on smithi131 2024-02-19T22:00:14.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:00:13.984106+0000 mgr.smithi131.cjubsp (mgr.14720) 313 : cephadm [INF] Upgrade: Updating prometheus.smithi131 2024-02-19T22:00:14.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:00:13.992111+0000 mgr.smithi131.cjubsp (mgr.14720) 314 : cephadm [INF] Deploying daemon prometheus.smithi131 on smithi131 2024-02-19T22:00:15.733 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:15 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:14.602911+0000 mgr.smithi131.cjubsp (mgr.14720) 315 : cluster [DBG] pgmap v221: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 4.7 KiB/s wr, 2 op/s 2024-02-19T22:00:15.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:15 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:14.602911+0000 mgr.smithi131.cjubsp (mgr.14720) 315 : cluster [DBG] pgmap v221: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 4.7 KiB/s wr, 2 op/s 2024-02-19T22:00:17.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:17 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:16.603438+0000 mgr.smithi131.cjubsp (mgr.14720) 316 : cluster [DBG] pgmap v222: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:17.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:17 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:16.603438+0000 mgr.smithi131.cjubsp (mgr.14720) 316 : cluster [DBG] pgmap v222: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:20.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:18.603933+0000 mgr.smithi131.cjubsp (mgr.14720) 317 : cluster [DBG] pgmap v223: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:20.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:19.240903+0000 mon.smithi131 (mon.0) 804 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:20.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:19.241650+0000 mon.smithi131 (mon.0) 805 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:20.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:18.603933+0000 mgr.smithi131.cjubsp (mgr.14720) 317 : cluster [DBG] pgmap v223: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:20.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:19.240903+0000 mon.smithi131 (mon.0) 804 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:20.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:19.241650+0000 mon.smithi131 (mon.0) 805 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:22.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:20.604430+0000 mgr.smithi131.cjubsp (mgr.14720) 318 : cluster [DBG] pgmap v224: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:22.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:20.604430+0000 mgr.smithi131.cjubsp (mgr.14720) 318 : cluster [DBG] pgmap v224: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:24.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:22.604927+0000 mgr.smithi131.cjubsp (mgr.14720) 319 : cluster [DBG] pgmap v225: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:24.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:22.604927+0000 mgr.smithi131.cjubsp (mgr.14720) 319 : cluster [DBG] pgmap v225: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.600846+0000 mon.smithi131 (mon.0) 806 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:24.605357+0000 mgr.smithi131.cjubsp (mgr.14720) 320 : cluster [DBG] pgmap v226: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.870315+0000 mon.smithi131 (mon.0) 807 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:00:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.870983+0000 mon.smithi131 (mon.0) 808 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:00:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.877313+0000 mon.smithi131 (mon.0) 809 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.884563+0000 mon.smithi131 (mon.0) 810 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:25.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.888878+0000 mon.smithi131 (mon.0) 811 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.889250+0000 mgr.smithi131.cjubsp (mgr.14720) 321 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.890741+0000 mon.smithi131 (mon.0) 812 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.892805+0000 mon.smithi131 (mon.0) 813 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.893967+0000 mon.smithi131 (mon.0) 814 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.895092+0000 mon.smithi131 (mon.0) 815 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.896873+0000 mon.smithi131 (mon.0) 816 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.898538+0000 mon.smithi131 (mon.0) 817 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.899435+0000 mon.smithi131 (mon.0) 818 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.900347+0000 mon.smithi131 (mon.0) 819 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.901293+0000 mon.smithi131 (mon.0) 820 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.902217+0000 mon.smithi131 (mon.0) 821 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.903322+0000 mon.smithi131 (mon.0) 822 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.904492+0000 mon.smithi131 (mon.0) 823 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:25.924 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:25 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:24.905646+0000 mon.smithi131 (mon.0) 824 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.600846+0000 mon.smithi131 (mon.0) 806 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:24.605357+0000 mgr.smithi131.cjubsp (mgr.14720) 320 : cluster [DBG] pgmap v226: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 426 B/s wr, 0 op/s 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.870315+0000 mon.smithi131 (mon.0) 807 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.870983+0000 mon.smithi131 (mon.0) 808 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.877313+0000 mon.smithi131 (mon.0) 809 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.884563+0000 mon.smithi131 (mon.0) 810 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.888878+0000 mon.smithi131 (mon.0) 811 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.889250+0000 mgr.smithi131.cjubsp (mgr.14720) 321 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.890741+0000 mon.smithi131 (mon.0) 812 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.892805+0000 mon.smithi131 (mon.0) 813 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.007 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.893967+0000 mon.smithi131 (mon.0) 814 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.895092+0000 mon.smithi131 (mon.0) 815 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.896873+0000 mon.smithi131 (mon.0) 816 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.898538+0000 mon.smithi131 (mon.0) 817 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.899435+0000 mon.smithi131 (mon.0) 818 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.900347+0000 mon.smithi131 (mon.0) 819 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.901293+0000 mon.smithi131 (mon.0) 820 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.902217+0000 mon.smithi131 (mon.0) 821 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.903322+0000 mon.smithi131 (mon.0) 822 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.904492+0000 mon.smithi131 (mon.0) 823 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:26.008 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:25 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:24.905646+0000 mon.smithi131 (mon.0) 824 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:27.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:00:26.416350+0000 mgr.smithi131.cjubsp (mgr.14720) 322 : cephadm [INF] Upgrade: Updating alertmanager.smithi131 2024-02-19T22:00:27.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:00:26.422567+0000 mgr.smithi131.cjubsp (mgr.14720) 323 : cephadm [INF] Deploying daemon alertmanager.smithi131 on smithi131 2024-02-19T22:00:27.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:27 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:26.605887+0000 mgr.smithi131.cjubsp (mgr.14720) 324 : cluster [DBG] pgmap v227: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:28.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:00:26.416350+0000 mgr.smithi131.cjubsp (mgr.14720) 322 : cephadm [INF] Upgrade: Updating alertmanager.smithi131 2024-02-19T22:00:28.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:00:26.422567+0000 mgr.smithi131.cjubsp (mgr.14720) 323 : cephadm [INF] Deploying daemon alertmanager.smithi131 on smithi131 2024-02-19T22:00:28.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:27 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:26.605887+0000 mgr.smithi131.cjubsp (mgr.14720) 324 : cluster [DBG] pgmap v227: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:29.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:29 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:28.606403+0000 mgr.smithi131.cjubsp (mgr.14720) 325 : cluster [DBG] pgmap v228: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:30.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:28.606403+0000 mgr.smithi131.cjubsp (mgr.14720) 325 : cluster [DBG] pgmap v228: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:31.038 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (13m) 7s ago 15m 21.0M - 0.20.0 0881eb8f169f aac70c469f59 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (5m) 7s ago 15m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (5m) 22s ago 14m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (13m) 7s ago 14m 35.0M - 6.7.4 557c83e11646 4f5341e6938d 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (75s) 7s ago 9m 36.9M - 16.2.14-548-g81bd20d6 823b921721f5 9ed7e8639d27 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (57s) 22s ago 9m 14.6M - 16.2.14-548-g81bd20d6 823b921721f5 b7eb2031d093 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (6m) 7s ago 16m 448M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (5m) 22s ago 13m 389M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (5m) 7s ago 16m 55.5M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T22:00:31.632 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (5m) 22s ago 13m 42.7M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (49s) 7s ago 6m 122M - 3.5 823b921721f5 b99130858f09 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (43s) 22s ago 6m 62.0M - 3.5 823b921721f5 faf5deba11ee 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (31s) 7s ago 14m 11.2M - 1.3.1 1dbe0e931976 9edb16b58848 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (25s) 22s ago 13m 3632k - 1.3.1 1dbe0e931976 b49f05a21958 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (5m) 7s ago 13m 123M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (4m) 7s ago 12m 113M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (4m) 7s ago 12m 92.3M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (3m) 7s ago 12m 121M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (2m) 22s ago 11m 128M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (2m) 22s ago 11m 119M 2712M 16.2.14-548-g81bd20d6 823b921721f5 4c4b41297954 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (119s) 22s ago 11m 113M 2712M 16.2.14-548-g81bd20d6 823b921721f5 e9be2f442ef3 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (90s) 22s ago 10m 138M 2712M 16.2.14-548-g81bd20d6 823b921721f5 10d1c2418ed8 2024-02-19T22:00:31.633 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (12s) 7s ago 14m 36.7M - 2.33.4 514e6a882f6e 64ccabaf7590 2024-02-19T22:00:31.843 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:30.607039+0000 mgr.smithi131.cjubsp (mgr.14720) 326 : cluster [DBG] pgmap v229: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:31.843 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:31 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:31.026985+0000 mgr.smithi131.cjubsp (mgr.14720) 327 : audit [DBG] from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:31.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:30.607039+0000 mgr.smithi131.cjubsp (mgr.14720) 326 : cluster [DBG] pgmap v229: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:31.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:31 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:31.026985+0000 mgr.smithi131.cjubsp (mgr.14720) 327 : audit [DBG] from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:31.981 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T22:00:31.981 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T22:00:31.981 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:00:31.981 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:00:31.981 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T22:00:31.981 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:00:31.981 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:00:31.981 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 8 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 14 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T22:00:31.982 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T22:00:32.275 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T22:00:32.275 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": true, 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [ 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "crash", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "osd", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "mon", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "nfs", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "mds" 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": "18/23 daemons upgraded", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "Currently upgrading alertmanager daemons", 2024-02-19T22:00:32.276 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T22:00:32.277 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T22:00:32.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:31.327872+0000 mgr.smithi131.cjubsp (mgr.14720) 328 : audit [DBG] from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:32.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:31.623846+0000 mgr.smithi131.cjubsp (mgr.14720) 329 : audit [DBG] from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:32.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:31.982294+0000 mon.smithi131 (mon.0) 825 : audit [DBG] from='client.? 172.21.15.131:0/3229073041' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:32.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:32.276303+0000 mgr.smithi131.cjubsp (mgr.14720) 330 : audit [DBG] from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:33.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:31.327872+0000 mgr.smithi131.cjubsp (mgr.14720) 328 : audit [DBG] from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:33.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:31.623846+0000 mgr.smithi131.cjubsp (mgr.14720) 329 : audit [DBG] from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:33.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:31.982294+0000 mon.smithi131 (mon.0) 825 : audit [DBG] from='client.? 172.21.15.131:0/3229073041' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:33.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:32.276303+0000 mgr.smithi131.cjubsp (mgr.14720) 330 : audit [DBG] from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:00:33.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:33 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:32.607704+0000 mgr.smithi131.cjubsp (mgr.14720) 331 : cluster [DBG] pgmap v230: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:33.923 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:33 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:32.607704+0000 mgr.smithi131.cjubsp (mgr.14720) 331 : cluster [DBG] pgmap v230: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:35.926 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:34.608362+0000 mgr.smithi131.cjubsp (mgr.14720) 332 : cluster [DBG] pgmap v231: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:00:36.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:34.608362+0000 mgr.smithi131.cjubsp (mgr.14720) 332 : cluster [DBG] pgmap v231: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:00:37.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:36.101370+0000 mon.smithi131 (mon.0) 826 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:37.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:36.102122+0000 mon.smithi131 (mon.0) 827 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:37.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:36.101370+0000 mon.smithi131 (mon.0) 826 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:37.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:36.102122+0000 mon.smithi131 (mon.0) 827 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:38.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:36.609143+0000 mgr.smithi131.cjubsp (mgr.14720) 333 : cluster [DBG] pgmap v232: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:38.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:36.609143+0000 mgr.smithi131.cjubsp (mgr.14720) 333 : cluster [DBG] pgmap v232: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:40.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:38.609908+0000 mgr.smithi131.cjubsp (mgr.14720) 334 : cluster [DBG] pgmap v233: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:40.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:38.609908+0000 mgr.smithi131.cjubsp (mgr.14720) 334 : cluster [DBG] pgmap v233: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:42.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:40.610434+0000 mgr.smithi131.cjubsp (mgr.14720) 335 : cluster [DBG] pgmap v234: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:42.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.829746+0000 mon.smithi131 (mon.0) 828 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:42.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.830600+0000 mon.smithi131 (mon.0) 829 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:00:42.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.831187+0000 mon.smithi131 (mon.0) 830 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:00:42.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.836692+0000 mon.smithi131 (mon.0) 831 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.843677+0000 mon.smithi131 (mon.0) 832 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.847818+0000 mon.smithi131 (mon.0) 833 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.848883+0000 mon.smithi131 (mon.0) 834 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi131.front.sepia.ceph.com:9093"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.854805+0000 mon.smithi131 (mon.0) 835 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.858086+0000 mon.smithi131 (mon.0) 836 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.861434+0000 mon.smithi131 (mon.0) 837 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.862998+0000 mon.smithi131 (mon.0) 838 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.864219+0000 mon.smithi131 (mon.0) 839 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.865938+0000 mon.smithi131 (mon.0) 840 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.867490+0000 mon.smithi131 (mon.0) 841 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.868460+0000 mon.smithi131 (mon.0) 842 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.869258+0000 mon.smithi131 (mon.0) 843 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.870456+0000 mon.smithi131 (mon.0) 844 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.871692+0000 mon.smithi131 (mon.0) 845 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.872863+0000 mon.smithi131 (mon.0) 846 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.874074+0000 mon.smithi131 (mon.0) 847 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.875088+0000 mon.smithi131 (mon.0) 848 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.876085+0000 mon.smithi131 (mon.0) 849 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:40.610434+0000 mgr.smithi131.cjubsp (mgr.14720) 335 : cluster [DBG] pgmap v234: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.829746+0000 mon.smithi131 (mon.0) 828 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.830600+0000 mon.smithi131 (mon.0) 829 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:00:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.831187+0000 mon.smithi131 (mon.0) 830 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:00:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.836692+0000 mon.smithi131 (mon.0) 831 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:42.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.843677+0000 mon.smithi131 (mon.0) 832 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.847818+0000 mon.smithi131 (mon.0) 833 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.848883+0000 mon.smithi131 (mon.0) 834 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi131.front.sepia.ceph.com:9093"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.854805+0000 mon.smithi131 (mon.0) 835 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.858086+0000 mon.smithi131 (mon.0) 836 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.861434+0000 mon.smithi131 (mon.0) 837 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.862998+0000 mon.smithi131 (mon.0) 838 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.864219+0000 mon.smithi131 (mon.0) 839 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.865938+0000 mon.smithi131 (mon.0) 840 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.867490+0000 mon.smithi131 (mon.0) 841 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.868460+0000 mon.smithi131 (mon.0) 842 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.869258+0000 mon.smithi131 (mon.0) 843 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.870456+0000 mon.smithi131 (mon.0) 844 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.871692+0000 mon.smithi131 (mon.0) 845 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.872863+0000 mon.smithi131 (mon.0) 846 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.874074+0000 mon.smithi131 (mon.0) 847 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.875088+0000 mon.smithi131 (mon.0) 848 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:42.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.876085+0000 mon.smithi131 (mon.0) 849 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:00:43.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.848136+0000 mgr.smithi131.cjubsp (mgr.14720) 336 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-19T22:00:43.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:41.849120+0000 mgr.smithi131.cjubsp (mgr.14720) 337 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi131.front.sepia.ceph.com:9093"}]: dispatch 2024-02-19T22:00:43.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.848136+0000 mgr.smithi131.cjubsp (mgr.14720) 336 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-19T22:00:43.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:41.849120+0000 mgr.smithi131.cjubsp (mgr.14720) 337 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi131.front.sepia.ceph.com:9093"}]: dispatch 2024-02-19T22:00:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:42.610948+0000 mgr.smithi131.cjubsp (mgr.14720) 338 : cluster [DBG] pgmap v235: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:00:43.279050+0000 mgr.smithi131.cjubsp (mgr.14720) 339 : cephadm [INF] Upgrade: Updating grafana.smithi131 2024-02-19T22:00:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:43.618585+0000 mon.smithi131 (mon.0) 850 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:43.621590+0000 mon.smithi131 (mon.0) 851 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:44.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:43.623907+0000 mon.smithi131 (mon.0) 852 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-19T22:00:44.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:42.610948+0000 mgr.smithi131.cjubsp (mgr.14720) 338 : cluster [DBG] pgmap v235: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:44.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:00:43.279050+0000 mgr.smithi131.cjubsp (mgr.14720) 339 : cephadm [INF] Upgrade: Updating grafana.smithi131 2024-02-19T22:00:44.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:43.618585+0000 mon.smithi131 (mon.0) 850 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:44.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:43.621590+0000 mon.smithi131 (mon.0) 851 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:44.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:43.623907+0000 mon.smithi131 (mon.0) 852 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-19T22:00:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:43.624261+0000 mgr.smithi131.cjubsp (mgr.14720) 340 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-19T22:00:45.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:00:43.628868+0000 mgr.smithi131.cjubsp (mgr.14720) 341 : cephadm [INF] Deploying daemon grafana.smithi131 on smithi131 2024-02-19T22:00:45.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:43.624261+0000 mgr.smithi131.cjubsp (mgr.14720) 340 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-19T22:00:45.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:00:43.628868+0000 mgr.smithi131.cjubsp (mgr.14720) 341 : cephadm [INF] Deploying daemon grafana.smithi131 on smithi131 2024-02-19T22:00:46.255 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:44.611465+0000 mgr.smithi131.cjubsp (mgr.14720) 342 : cluster [DBG] pgmap v236: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:00:46.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:44.611465+0000 mgr.smithi131.cjubsp (mgr.14720) 342 : cluster [DBG] pgmap v236: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:00:48.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:46.611967+0000 mgr.smithi131.cjubsp (mgr.14720) 343 : cluster [DBG] pgmap v237: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:48.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:46.611967+0000 mgr.smithi131.cjubsp (mgr.14720) 343 : cluster [DBG] pgmap v237: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:50.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:48.612396+0000 mgr.smithi131.cjubsp (mgr.14720) 344 : cluster [DBG] pgmap v238: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:50.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:48.612396+0000 mgr.smithi131.cjubsp (mgr.14720) 344 : cluster [DBG] pgmap v238: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:52.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:50.612939+0000 mgr.smithi131.cjubsp (mgr.14720) 345 : cluster [DBG] pgmap v239: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:52.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:50.612939+0000 mgr.smithi131.cjubsp (mgr.14720) 345 : cluster [DBG] pgmap v239: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:54.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:52.613420+0000 mgr.smithi131.cjubsp (mgr.14720) 346 : cluster [DBG] pgmap v240: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:54.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:52.613420+0000 mgr.smithi131.cjubsp (mgr.14720) 346 : cluster [DBG] pgmap v240: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:56.217 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:54.613951+0000 mgr.smithi131.cjubsp (mgr.14720) 347 : cluster [DBG] pgmap v241: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:00:56.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:54.613951+0000 mgr.smithi131.cjubsp (mgr.14720) 347 : cluster [DBG] pgmap v241: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:00:58.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:56.614420+0000 mgr.smithi131.cjubsp (mgr.14720) 348 : cluster [DBG] pgmap v242: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:58.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:57.727135+0000 mon.smithi131 (mon.0) 853 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:58.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:00:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:00:57.727970+0000 mon.smithi131 (mon.0) 854 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:00:58.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:56.614420+0000 mgr.smithi131.cjubsp (mgr.14720) 348 : cluster [DBG] pgmap v242: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:00:58.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:57.727135+0000 mon.smithi131 (mon.0) 853 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:00:58.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:00:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:00:57.727970+0000 mon.smithi131 (mon.0) 854 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:01:00.256 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:00:58.614865+0000 mgr.smithi131.cjubsp (mgr.14720) 349 : cluster [DBG] pgmap v243: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:00.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:00:58.614865+0000 mgr.smithi131.cjubsp (mgr.14720) 349 : cluster [DBG] pgmap v243: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:02.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:00.615383+0000 mgr.smithi131.cjubsp (mgr.14720) 350 : cluster [DBG] pgmap v244: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:02.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:00.615383+0000 mgr.smithi131.cjubsp (mgr.14720) 350 : cluster [DBG] pgmap v244: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:02.590 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (27s) 21s ago 15m 13.3M - 0.23.0 ba2b418f427c d1bf67e22514 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (5m) 21s ago 15m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (5m) 53s ago 14m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 starting - - - - 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (107s) 21s ago 9m 36.9M - 16.2.14-548-g81bd20d6 823b921721f5 9ed7e8639d27 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (88s) 53s ago 9m 14.6M - 16.2.14-548-g81bd20d6 823b921721f5 b7eb2031d093 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (6m) 21s ago 16m 449M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (6m) 53s ago 14m 389M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (6m) 21s ago 17m 55.8M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (6m) 53s ago 14m 42.7M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (80s) 21s ago 7m 125M - 3.5 823b921721f5 b99130858f09 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (75s) 53s ago 7m 62.0M - 3.5 823b921721f5 faf5deba11ee 2024-02-19T22:01:03.213 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (63s) 21s ago 15m 12.3M - 1.3.1 1dbe0e931976 9edb16b58848 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (57s) 53s ago 14m 3632k - 1.3.1 1dbe0e931976 b49f05a21958 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (5m) 21s ago 13m 123M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (5m) 21s ago 13m 113M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (4m) 21s ago 13m 92.4M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (3m) 21s ago 12m 121M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (3m) 53s ago 12m 128M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (2m) 53s ago 12m 119M 2712M 16.2.14-548-g81bd20d6 823b921721f5 4c4b41297954 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (2m) 53s ago 11m 113M 2712M 16.2.14-548-g81bd20d6 823b921721f5 e9be2f442ef3 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (2m) 53s ago 11m 138M 2712M 16.2.14-548-g81bd20d6 823b921721f5 10d1c2418ed8 2024-02-19T22:01:03.214 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (44s) 21s ago 14m 62.6M - 2.33.4 514e6a882f6e 64ccabaf7590 2024-02-19T22:01:03.566 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T22:01:03.566 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 8 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 14 2024-02-19T22:01:03.567 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T22:01:03.568 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T22:01:03.878 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T22:01:03.879 INFO:teuthology.orchestra.run.smithi131.stdout: "target_image": null, 2024-02-19T22:01:03.879 INFO:teuthology.orchestra.run.smithi131.stdout: "in_progress": false, 2024-02-19T22:01:03.879 INFO:teuthology.orchestra.run.smithi131.stdout: "which": "", 2024-02-19T22:01:03.879 INFO:teuthology.orchestra.run.smithi131.stdout: "services_complete": [], 2024-02-19T22:01:03.879 INFO:teuthology.orchestra.run.smithi131.stdout: "progress": null, 2024-02-19T22:01:03.879 INFO:teuthology.orchestra.run.smithi131.stdout: "message": "", 2024-02-19T22:01:03.879 INFO:teuthology.orchestra.run.smithi131.stdout: "is_paused": false 2024-02-19T22:01:03.879 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:02.576829+0000 mgr.smithi131.cjubsp (mgr.14720) 351 : audit [DBG] from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:02.615873+0000 mgr.smithi131.cjubsp (mgr.14720) 352 : cluster [DBG] pgmap v245: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:02.910874+0000 mgr.smithi131.cjubsp (mgr.14720) 353 : audit [DBG] from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.204842+0000 mgr.smithi131.cjubsp (mgr.14720) 354 : audit [DBG] from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.567359+0000 mon.smithi131 (mon.0) 855 : audit [DBG] from='client.? 172.21.15.131:0/1855595119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.606409+0000 mon.smithi131 (mon.0) 856 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.607336+0000 mon.smithi131 (mon.0) 857 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.608052+0000 mon.smithi131 (mon.0) 858 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.613525+0000 mon.smithi131 (mon.0) 859 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.424 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.620191+0000 mon.smithi131 (mon.0) 860 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.624199+0000 mon.smithi131 (mon.0) 861 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.625545+0000 mon.smithi131 (mon.0) 862 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi131.front.sepia.ceph.com:3000"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.630947+0000 mon.smithi131 (mon.0) 863 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.633361+0000 mon.smithi131 (mon.0) 864 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.638308+0000 mon.smithi131 (mon.0) 865 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.639653+0000 mon.smithi131 (mon.0) 866 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.640820+0000 mon.smithi131 (mon.0) 867 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.642451+0000 mon.smithi131 (mon.0) 868 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.644210+0000 mon.smithi131 (mon.0) 869 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.645236+0000 mon.smithi131 (mon.0) 870 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.646240+0000 mon.smithi131 (mon.0) 871 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.647510+0000 mon.smithi131 (mon.0) 872 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.648722+0000 mon.smithi131 (mon.0) 873 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.425 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.650026+0000 mon.smithi131 (mon.0) 874 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.651650+0000 mon.smithi131 (mon.0) 875 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.653015+0000 mon.smithi131 (mon.0) 876 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.654409+0000 mon.smithi131 (mon.0) 877 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.655746+0000 mon.smithi131 (mon.0) 878 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.660618+0000 mon.smithi131 (mon.0) 879 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.661366+0000 mon.smithi131 (mon.0) 880 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.664732+0000 mon.smithi131 (mon.0) 881 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.665478+0000 mon.smithi131 (mon.0) 882 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.668890+0000 mon.smithi131 (mon.0) 883 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.669612+0000 mon.smithi131 (mon.0) 884 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.672931+0000 mon.smithi131 (mon.0) 885 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.673583+0000 mon.smithi131 (mon.0) 886 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.676688+0000 mon.smithi131 (mon.0) 887 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-19T22:01:04.426 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.677308+0000 mon.smithi131 (mon.0) 888 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.680469+0000 mon.smithi131 (mon.0) 889 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.681140+0000 mon.smithi131 (mon.0) 890 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.683956+0000 mon.smithi131 (mon.0) 891 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.684764+0000 mon.smithi131 (mon.0) 892 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.688427+0000 mon.smithi131 (mon.0) 893 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.689327+0000 mon.smithi131 (mon.0) 894 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.694820+0000 mon.smithi131 (mon.0) 895 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.698632+0000 mon.smithi131 (mon.0) 896 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.699313+0000 mon.smithi131 (mon.0) 897 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.702171+0000 mon.smithi131 (mon.0) 898 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.702697+0000 mon.smithi131 (mon.0) 899 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.703148+0000 mon.smithi131 (mon.0) 900 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.703567+0000 mon.smithi131 (mon.0) 901 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.704001+0000 mon.smithi131 (mon.0) 902 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.427 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.704424+0000 mon.smithi131 (mon.0) 903 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.707350+0000 mon.smithi131 (mon.0) 904 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.708186+0000 mon.smithi131 (mon.0) 905 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.710277+0000 mon.smithi131 (mon.0) 906 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.710681+0000 mon.smithi131 (mon.0) 907 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.711886+0000 mon.smithi131 (mon.0) 908 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.712393+0000 mon.smithi131 (mon.0) 909 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.716371+0000 mon.smithi131 (mon.0) 910 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.721742+0000 mon.smithi131 (mon.0) 911 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.724412+0000 mon.smithi131 (mon.0) 912 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.725524+0000 mon.smithi131 (mon.0) 913 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:01:04.428 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.726042+0000 mon.smithi131 (mon.0) 914 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:01:04.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.729919+0000 mon.smithi131 (mon.0) 915 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.429 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.735095+0000 mon.smithi131 (mon.0) 916 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:02.576829+0000 mgr.smithi131.cjubsp (mgr.14720) 351 : audit [DBG] from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:02.615873+0000 mgr.smithi131.cjubsp (mgr.14720) 352 : cluster [DBG] pgmap v245: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:02.910874+0000 mgr.smithi131.cjubsp (mgr.14720) 353 : audit [DBG] from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.204842+0000 mgr.smithi131.cjubsp (mgr.14720) 354 : audit [DBG] from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.567359+0000 mon.smithi131 (mon.0) 855 : audit [DBG] from='client.? 172.21.15.131:0/1855595119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.606409+0000 mon.smithi131 (mon.0) 856 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.607336+0000 mon.smithi131 (mon.0) 857 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.608052+0000 mon.smithi131 (mon.0) 858 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.613525+0000 mon.smithi131 (mon.0) 859 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.620191+0000 mon.smithi131 (mon.0) 860 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.624199+0000 mon.smithi131 (mon.0) 861 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.625545+0000 mon.smithi131 (mon.0) 862 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi131.front.sepia.ceph.com:3000"}]: dispatch 2024-02-19T22:01:04.507 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.630947+0000 mon.smithi131 (mon.0) 863 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.633361+0000 mon.smithi131 (mon.0) 864 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.638308+0000 mon.smithi131 (mon.0) 865 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.639653+0000 mon.smithi131 (mon.0) 866 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.640820+0000 mon.smithi131 (mon.0) 867 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.642451+0000 mon.smithi131 (mon.0) 868 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.644210+0000 mon.smithi131 (mon.0) 869 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.645236+0000 mon.smithi131 (mon.0) 870 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.646240+0000 mon.smithi131 (mon.0) 871 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.647510+0000 mon.smithi131 (mon.0) 872 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.648722+0000 mon.smithi131 (mon.0) 873 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.650026+0000 mon.smithi131 (mon.0) 874 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.651650+0000 mon.smithi131 (mon.0) 875 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.653015+0000 mon.smithi131 (mon.0) 876 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.508 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.654409+0000 mon.smithi131 (mon.0) 877 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.655746+0000 mon.smithi131 (mon.0) 878 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.660618+0000 mon.smithi131 (mon.0) 879 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.661366+0000 mon.smithi131 (mon.0) 880 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.664732+0000 mon.smithi131 (mon.0) 881 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.665478+0000 mon.smithi131 (mon.0) 882 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.668890+0000 mon.smithi131 (mon.0) 883 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.669612+0000 mon.smithi131 (mon.0) 884 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.672931+0000 mon.smithi131 (mon.0) 885 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.673583+0000 mon.smithi131 (mon.0) 886 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.676688+0000 mon.smithi131 (mon.0) 887 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.677308+0000 mon.smithi131 (mon.0) 888 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.680469+0000 mon.smithi131 (mon.0) 889 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.681140+0000 mon.smithi131 (mon.0) 890 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-19T22:01:04.509 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.683956+0000 mon.smithi131 (mon.0) 891 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.684764+0000 mon.smithi131 (mon.0) 892 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.688427+0000 mon.smithi131 (mon.0) 893 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.689327+0000 mon.smithi131 (mon.0) 894 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.694820+0000 mon.smithi131 (mon.0) 895 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.698632+0000 mon.smithi131 (mon.0) 896 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.699313+0000 mon.smithi131 (mon.0) 897 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.702171+0000 mon.smithi131 (mon.0) 898 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.702697+0000 mon.smithi131 (mon.0) 899 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.703148+0000 mon.smithi131 (mon.0) 900 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.703567+0000 mon.smithi131 (mon.0) 901 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.704001+0000 mon.smithi131 (mon.0) 902 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.704424+0000 mon.smithi131 (mon.0) 903 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-02-19T22:01:04.510 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.707350+0000 mon.smithi131 (mon.0) 904 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.708186+0000 mon.smithi131 (mon.0) 905 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.710277+0000 mon.smithi131 (mon.0) 906 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.710681+0000 mon.smithi131 (mon.0) 907 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.711886+0000 mon.smithi131 (mon.0) 908 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.712393+0000 mon.smithi131 (mon.0) 909 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.716371+0000 mon.smithi131 (mon.0) 910 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.721742+0000 mon.smithi131 (mon.0) 911 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.724412+0000 mon.smithi131 (mon.0) 912 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.725524+0000 mon.smithi131 (mon.0) 913 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.726042+0000 mon.smithi131 (mon.0) 914 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.729919+0000 mon.smithi131 (mon.0) 915 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:04.511 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.735095+0000 mon.smithi131 (mon.0) 916 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.624714+0000 mgr.smithi131.cjubsp (mgr.14720) 355 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-19T22:01:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.625947+0000 mgr.smithi131.cjubsp (mgr.14720) 356 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi131.front.sepia.ceph.com:3000"}]: dispatch 2024-02-19T22:01:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:01:03.656468+0000 mgr.smithi131.cjubsp (mgr.14720) 357 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-02-19T22:01:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cephadm 2024-02-19T22:01:03.707798+0000 mgr.smithi131.cjubsp (mgr.14720) 358 : cephadm [INF] Upgrade: Complete! 2024-02-19T22:01:05.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:03.879640+0000 mgr.smithi131.cjubsp (mgr.14720) 359 : audit [DBG] from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.624714+0000 mgr.smithi131.cjubsp (mgr.14720) 355 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-19T22:01:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.625947+0000 mgr.smithi131.cjubsp (mgr.14720) 356 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi131.front.sepia.ceph.com:3000"}]: dispatch 2024-02-19T22:01:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:01:03.656468+0000 mgr.smithi131.cjubsp (mgr.14720) 357 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-02-19T22:01:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cephadm 2024-02-19T22:01:03.707798+0000 mgr.smithi131.cjubsp (mgr.14720) 358 : cephadm [INF] Upgrade: Complete! 2024-02-19T22:01:05.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:03.879640+0000 mgr.smithi131.cjubsp (mgr.14720) 359 : audit [DBG] from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:04.616347+0000 mgr.smithi131.cjubsp (mgr.14720) 360 : cluster [DBG] pgmap v246: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:06.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:05.949570+0000 mon.smithi131 (mon.0) 917 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:04.616347+0000 mgr.smithi131.cjubsp (mgr.14720) 360 : cluster [DBG] pgmap v246: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:06.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:05.949570+0000 mon.smithi131 (mon.0) 917 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:01:08.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:06.617087+0000 mgr.smithi131.cjubsp (mgr.14720) 361 : cluster [DBG] pgmap v247: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:08.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:06.617087+0000 mgr.smithi131.cjubsp (mgr.14720) 361 : cluster [DBG] pgmap v247: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:10.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:08.617778+0000 mgr.smithi131.cjubsp (mgr.14720) 362 : cluster [DBG] pgmap v248: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:10.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:08.617778+0000 mgr.smithi131.cjubsp (mgr.14720) 362 : cluster [DBG] pgmap v248: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:12.423 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:10.618433+0000 mgr.smithi131.cjubsp (mgr.14720) 363 : cluster [DBG] pgmap v249: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:12.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:10.618433+0000 mgr.smithi131.cjubsp (mgr.14720) 363 : cluster [DBG] pgmap v249: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:14.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:12.618977+0000 mgr.smithi131.cjubsp (mgr.14720) 364 : cluster [DBG] pgmap v250: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:14.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:12.618977+0000 mgr.smithi131.cjubsp (mgr.14720) 364 : cluster [DBG] pgmap v250: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:16.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:14.619551+0000 mgr.smithi131.cjubsp (mgr.14720) 365 : cluster [DBG] pgmap v251: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:16.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:14.619551+0000 mgr.smithi131.cjubsp (mgr.14720) 365 : cluster [DBG] pgmap v251: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:18.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:16.620317+0000 mgr.smithi131.cjubsp (mgr.14720) 366 : cluster [DBG] pgmap v252: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:18.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:16.620317+0000 mgr.smithi131.cjubsp (mgr.14720) 366 : cluster [DBG] pgmap v252: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:20.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:18.620989+0000 mgr.smithi131.cjubsp (mgr.14720) 367 : cluster [DBG] pgmap v253: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:20.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:18.620989+0000 mgr.smithi131.cjubsp (mgr.14720) 367 : cluster [DBG] pgmap v253: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:22.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:20.621579+0000 mgr.smithi131.cjubsp (mgr.14720) 368 : cluster [DBG] pgmap v254: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:22.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:20.621579+0000 mgr.smithi131.cjubsp (mgr.14720) 368 : cluster [DBG] pgmap v254: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:24.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:22.622139+0000 mgr.smithi131.cjubsp (mgr.14720) 369 : cluster [DBG] pgmap v255: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:24.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:22.622139+0000 mgr.smithi131.cjubsp (mgr.14720) 369 : cluster [DBG] pgmap v255: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:26.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:24.622736+0000 mgr.smithi131.cjubsp (mgr.14720) 370 : cluster [DBG] pgmap v256: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:26.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:24.622736+0000 mgr.smithi131.cjubsp (mgr.14720) 370 : cluster [DBG] pgmap v256: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:28.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:26.623344+0000 mgr.smithi131.cjubsp (mgr.14720) 371 : cluster [DBG] pgmap v257: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:28.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:26.623344+0000 mgr.smithi131.cjubsp (mgr.14720) 371 : cluster [DBG] pgmap v257: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:30.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:28.623897+0000 mgr.smithi131.cjubsp (mgr.14720) 372 : cluster [DBG] pgmap v258: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:28.623897+0000 mgr.smithi131.cjubsp (mgr.14720) 372 : cluster [DBG] pgmap v258: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:32.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:30.624443+0000 mgr.smithi131.cjubsp (mgr.14720) 373 : cluster [DBG] pgmap v259: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:30.624443+0000 mgr.smithi131.cjubsp (mgr.14720) 373 : cluster [DBG] pgmap v259: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:34.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:32.625125+0000 mgr.smithi131.cjubsp (mgr.14720) 374 : cluster [DBG] pgmap v260: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:34.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:32.625125+0000 mgr.smithi131.cjubsp (mgr.14720) 374 : cluster [DBG] pgmap v260: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:34.980 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph orch ps' 2024-02-19T22:01:35.491 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:35 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:34.183375+0000 mgr.smithi131.cjubsp (mgr.14720) 375 : audit [DBG] from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:35.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:35 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:34.183375+0000 mgr.smithi131.cjubsp (mgr.14720) 375 : audit [DBG] from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:36.327 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:34.625657+0000 mgr.smithi131.cjubsp (mgr.14720) 376 : cluster [DBG] pgmap v261: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:36.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:34.625657+0000 mgr.smithi131.cjubsp (mgr.14720) 376 : cluster [DBG] pgmap v261: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:37.067 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (61s) 33s ago 16m 13.5M - 0.23.0 ba2b418f427c d1bf67e22514 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (6m) 33s ago 16m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (6m) 87s ago 15m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (39s) 33s ago 15m 31.2M - 8.3.5 dad864ee21e9 07ea7550dea6 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (2m) 33s ago 10m 37.0M - 16.2.14-548-g81bd20d6 823b921721f5 9ed7e8639d27 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (2m) 87s ago 10m 14.6M - 16.2.14-548-g81bd20d6 823b921721f5 b7eb2031d093 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (7m) 33s ago 17m 450M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (7m) 87s ago 15m 389M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (6m) 33s ago 17m 56.0M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (6m) 87s ago 15m 42.7M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (114s) 33s ago 7m 135M - 3.5 823b921721f5 b99130858f09 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (108s) 87s ago 7m 62.0M - 3.5 823b921721f5 faf5deba11ee 2024-02-19T22:01:37.068 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (97s) 33s ago 15m 12.2M - 1.3.1 1dbe0e931976 9edb16b58848 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (90s) 87s ago 14m 3632k - 1.3.1 1dbe0e931976 b49f05a21958 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (6m) 33s ago 14m 123M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (5m) 33s ago 14m 113M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (5m) 33s ago 13m 92.4M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (4m) 33s ago 13m 121M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (4m) 87s ago 12m 128M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (3m) 87s ago 12m 119M 2712M 16.2.14-548-g81bd20d6 823b921721f5 4c4b41297954 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (3m) 87s ago 12m 113M 2712M 16.2.14-548-g81bd20d6 823b921721f5 e9be2f442ef3 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (2m) 87s ago 11m 138M 2712M 16.2.14-548-g81bd20d6 823b921721f5 10d1c2418ed8 2024-02-19T22:01:37.069 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (78s) 33s ago 15m 62.7M - 2.33.4 514e6a882f6e 64ccabaf7590 2024-02-19T22:01:37.686 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph versions' 2024-02-19T22:01:38.453 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:36.626166+0000 mgr.smithi131.cjubsp (mgr.14720) 377 : cluster [DBG] pgmap v262: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:38.453 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:38 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:37.059341+0000 mgr.smithi131.cjubsp (mgr.14720) 378 : audit [DBG] from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:36.626166+0000 mgr.smithi131.cjubsp (mgr.14720) 377 : cluster [DBG] pgmap v262: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:38.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:38 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:37.059341+0000 mgr.smithi131.cjubsp (mgr.14720) 378 : audit [DBG] from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:01:39.872 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T22:01:39.872 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T22:01:39.872 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:01:39.872 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 8 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 14 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T22:01:39.873 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T22:01:40.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:38.626944+0000 mgr.smithi131.cjubsp (mgr.14720) 379 : cluster [DBG] pgmap v263: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:40.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:40 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:01:39.873174+0000 mon.smithi131 (mon.0) 918 : audit [DBG] from='client.? 172.21.15.131:0/3745158235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:40.558 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-02-19T22:01:40.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:38.626944+0000 mgr.smithi131.cjubsp (mgr.14720) 379 : cluster [DBG] pgmap v263: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:40.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:01:39.873174+0000 mon.smithi131 (mon.0) 918 : audit [DBG] from='client.? 172.21.15.131:0/3745158235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:01:42.393 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:42 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:40.627436+0000 mgr.smithi131.cjubsp (mgr.14720) 380 : cluster [DBG] pgmap v264: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:42.634 INFO:teuthology.orchestra.run.smithi131.stdout:wait for servicemap items w/ changing names to refresh 2024-02-19T22:01:42.672 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:40.627436+0000 mgr.smithi131.cjubsp (mgr.14720) 380 : cluster [DBG] pgmap v264: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:43.241 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'sleep 60' 2024-02-19T22:01:44.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:44 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:42.628166+0000 mgr.smithi131.cjubsp (mgr.14720) 381 : cluster [DBG] pgmap v265: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:44.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:42.628166+0000 mgr.smithi131.cjubsp (mgr.14720) 381 : cluster [DBG] pgmap v265: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:46.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:46 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:44.628672+0000 mgr.smithi131.cjubsp (mgr.14720) 382 : cluster [DBG] pgmap v266: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:46.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:44.628672+0000 mgr.smithi131.cjubsp (mgr.14720) 382 : cluster [DBG] pgmap v266: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:48.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:46.629471+0000 mgr.smithi131.cjubsp (mgr.14720) 383 : cluster [DBG] pgmap v267: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:48.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:46.629471+0000 mgr.smithi131.cjubsp (mgr.14720) 383 : cluster [DBG] pgmap v267: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:50.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:50 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:48.630122+0000 mgr.smithi131.cjubsp (mgr.14720) 384 : cluster [DBG] pgmap v268: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:50.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:48.630122+0000 mgr.smithi131.cjubsp (mgr.14720) 384 : cluster [DBG] pgmap v268: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:52.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:52 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:50.630799+0000 mgr.smithi131.cjubsp (mgr.14720) 385 : cluster [DBG] pgmap v269: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:52.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:50.630799+0000 mgr.smithi131.cjubsp (mgr.14720) 385 : cluster [DBG] pgmap v269: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:54.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:54 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:52.631266+0000 mgr.smithi131.cjubsp (mgr.14720) 386 : cluster [DBG] pgmap v270: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:54.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:52.631266+0000 mgr.smithi131.cjubsp (mgr.14720) 386 : cluster [DBG] pgmap v270: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:56.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:54.631848+0000 mgr.smithi131.cjubsp (mgr.14720) 387 : cluster [DBG] pgmap v271: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:56.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:54.631848+0000 mgr.smithi131.cjubsp (mgr.14720) 387 : cluster [DBG] pgmap v271: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:01:58.505 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:01:58 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:56.632403+0000 mgr.smithi131.cjubsp (mgr.14720) 388 : cluster [DBG] pgmap v272: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:01:58.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:01:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:56.632403+0000 mgr.smithi131.cjubsp (mgr.14720) 388 : cluster [DBG] pgmap v272: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:00.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:01:58.633014+0000 mgr.smithi131.cjubsp (mgr.14720) 389 : cluster [DBG] pgmap v273: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:00.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:01:58.633014+0000 mgr.smithi131.cjubsp (mgr.14720) 389 : cluster [DBG] pgmap v273: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:02.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:00.633762+0000 mgr.smithi131.cjubsp (mgr.14720) 390 : cluster [DBG] pgmap v274: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:02.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:02 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:00.633762+0000 mgr.smithi131.cjubsp (mgr.14720) 390 : cluster [DBG] pgmap v274: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:04.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:02.634269+0000 mgr.smithi131.cjubsp (mgr.14720) 391 : cluster [DBG] pgmap v275: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:04.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:03.737961+0000 mon.smithi131 (mon.0) 919 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:02:04.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:04.034152+0000 mon.smithi131 (mon.0) 920 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:02:04.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:04.034769+0000 mon.smithi131 (mon.0) 921 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:02:04.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:04.038857+0000 mon.smithi131 (mon.0) 922 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:02:04.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:04.044123+0000 mon.smithi131 (mon.0) 923 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:02:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:02.634269+0000 mgr.smithi131.cjubsp (mgr.14720) 391 : cluster [DBG] pgmap v275: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:03.737961+0000 mon.smithi131 (mon.0) 919 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:02:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:04.034152+0000 mon.smithi131 (mon.0) 920 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:02:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:04.034769+0000 mon.smithi131 (mon.0) 921 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:02:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:04.038857+0000 mon.smithi131 (mon.0) 922 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:02:04.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:04.044123+0000 mon.smithi131 (mon.0) 923 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:02:06.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:04.634931+0000 mgr.smithi131.cjubsp (mgr.14720) 392 : cluster [DBG] pgmap v276: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:06.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:04.634931+0000 mgr.smithi131.cjubsp (mgr.14720) 392 : cluster [DBG] pgmap v276: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:08.660 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:08 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:06.635704+0000 mgr.smithi131.cjubsp (mgr.14720) 393 : cluster [DBG] pgmap v277: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:08.672 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:06.635704+0000 mgr.smithi131.cjubsp (mgr.14720) 393 : cluster [DBG] pgmap v277: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:10.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:08.636135+0000 mgr.smithi131.cjubsp (mgr.14720) 394 : cluster [DBG] pgmap v278: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:10.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:10 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:08.636135+0000 mgr.smithi131.cjubsp (mgr.14720) 394 : cluster [DBG] pgmap v278: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:12.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:10.636901+0000 mgr.smithi131.cjubsp (mgr.14720) 395 : cluster [DBG] pgmap v279: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:12.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:10.636901+0000 mgr.smithi131.cjubsp (mgr.14720) 395 : cluster [DBG] pgmap v279: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:14.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:14 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:12.637508+0000 mgr.smithi131.cjubsp (mgr.14720) 396 : cluster [DBG] pgmap v280: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:14.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:12.637508+0000 mgr.smithi131.cjubsp (mgr.14720) 396 : cluster [DBG] pgmap v280: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:16.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:16 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:14.638065+0000 mgr.smithi131.cjubsp (mgr.14720) 397 : cluster [DBG] pgmap v281: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:16.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:16 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:14.638065+0000 mgr.smithi131.cjubsp (mgr.14720) 397 : cluster [DBG] pgmap v281: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:18.660 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:18 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:16.638635+0000 mgr.smithi131.cjubsp (mgr.14720) 398 : cluster [DBG] pgmap v282: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:18.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:18 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:16.638635+0000 mgr.smithi131.cjubsp (mgr.14720) 398 : cluster [DBG] pgmap v282: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:20.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:20 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:18.639318+0000 mgr.smithi131.cjubsp (mgr.14720) 399 : cluster [DBG] pgmap v283: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:20.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:20 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:18.639318+0000 mgr.smithi131.cjubsp (mgr.14720) 399 : cluster [DBG] pgmap v283: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:22.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:22 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:20.640000+0000 mgr.smithi131.cjubsp (mgr.14720) 400 : cluster [DBG] pgmap v284: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:22.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:20.640000+0000 mgr.smithi131.cjubsp (mgr.14720) 400 : cluster [DBG] pgmap v284: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:24.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:24 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:22.640471+0000 mgr.smithi131.cjubsp (mgr.14720) 401 : cluster [DBG] pgmap v285: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:24.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:24 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:22.640471+0000 mgr.smithi131.cjubsp (mgr.14720) 401 : cluster [DBG] pgmap v285: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:26.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:26 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:24.641077+0000 mgr.smithi131.cjubsp (mgr.14720) 402 : cluster [DBG] pgmap v286: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:26.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:26 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:24.641077+0000 mgr.smithi131.cjubsp (mgr.14720) 402 : cluster [DBG] pgmap v286: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:28.660 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:28 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:26.641647+0000 mgr.smithi131.cjubsp (mgr.14720) 403 : cluster [DBG] pgmap v287: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:28.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:28 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:26.641647+0000 mgr.smithi131.cjubsp (mgr.14720) 403 : cluster [DBG] pgmap v287: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:30.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:30 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:28.642179+0000 mgr.smithi131.cjubsp (mgr.14720) 404 : cluster [DBG] pgmap v288: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:30.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:30 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:28.642179+0000 mgr.smithi131.cjubsp (mgr.14720) 404 : cluster [DBG] pgmap v288: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:32.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:32 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:30.642937+0000 mgr.smithi131.cjubsp (mgr.14720) 405 : cluster [DBG] pgmap v289: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:32.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:32 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:30.642937+0000 mgr.smithi131.cjubsp (mgr.14720) 405 : cluster [DBG] pgmap v289: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:34.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:34 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:32.643557+0000 mgr.smithi131.cjubsp (mgr.14720) 406 : cluster [DBG] pgmap v290: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:34.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:34 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:32.643557+0000 mgr.smithi131.cjubsp (mgr.14720) 406 : cluster [DBG] pgmap v290: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:36.674 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:36 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:34.644145+0000 mgr.smithi131.cjubsp (mgr.14720) 407 : cluster [DBG] pgmap v291: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:36.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:36 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:34.644145+0000 mgr.smithi131.cjubsp (mgr.14720) 407 : cluster [DBG] pgmap v291: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:37.675 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:37 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:36.644724+0000 mgr.smithi131.cjubsp (mgr.14720) 408 : cluster [DBG] pgmap v292: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:37.755 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:36.644724+0000 mgr.smithi131.cjubsp (mgr.14720) 408 : cluster [DBG] pgmap v292: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:40.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:39 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:38.645295+0000 mgr.smithi131.cjubsp (mgr.14720) 409 : cluster [DBG] pgmap v293: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:40.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:39 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:38.645295+0000 mgr.smithi131.cjubsp (mgr.14720) 409 : cluster [DBG] pgmap v293: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:42.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:41 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:40.646071+0000 mgr.smithi131.cjubsp (mgr.14720) 410 : cluster [DBG] pgmap v294: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:42.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:40.646071+0000 mgr.smithi131.cjubsp (mgr.14720) 410 : cluster [DBG] pgmap v294: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:44.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:43 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:42.646716+0000 mgr.smithi131.cjubsp (mgr.14720) 411 : cluster [DBG] pgmap v295: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:44.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:42.646716+0000 mgr.smithi131.cjubsp (mgr.14720) 411 : cluster [DBG] pgmap v295: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:45.810 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph orch ps' 2024-02-19T22:02:46.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:45 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:44.647325+0000 mgr.smithi131.cjubsp (mgr.14720) 412 : cluster [DBG] pgmap v296: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:46.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:44.647325+0000 mgr.smithi131.cjubsp (mgr.14720) 412 : cluster [DBG] pgmap v296: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:47.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:47 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:46.648098+0000 mgr.smithi131.cjubsp (mgr.14720) 413 : cluster [DBG] pgmap v297: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:47.952 INFO:teuthology.orchestra.run.smithi131.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:alertmanager.smithi131 smithi131 *:9093,9094 running (2m) 104s ago 17m 13.5M - 0.23.0 ba2b418f427c d1bf67e22514 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi131 smithi131 running (7m) 104s ago 17m 6631k - 16.2.14-548-g81bd20d6 823b921721f5 8d86e49cac93 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:crash.smithi196 smithi196 running (7m) 2m ago 16m 6643k - 16.2.14-548-g81bd20d6 823b921721f5 dc19348c7caa 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:grafana.smithi131 smithi131 *:3000 running (110s) 104s ago 16m 31.2M - 8.3.5 dad864ee21e9 07ea7550dea6 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi131.gemwnz smithi131 running (3m) 104s ago 11m 37.0M - 16.2.14-548-g81bd20d6 823b921721f5 9ed7e8639d27 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:mds.foofs.smithi196.hhhkwt smithi196 running (3m) 2m ago 11m 14.6M - 16.2.14-548-g81bd20d6 823b921721f5 b7eb2031d093 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi131.cjubsp smithi131 *:8443,9283 running (8m) 104s ago 18m 450M - 16.2.14-548-g81bd20d6 823b921721f5 3ced280cc96d 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:mgr.smithi196.voprex smithi196 *:8443,9283 running (8m) 2m ago 16m 389M - 16.2.14-548-g81bd20d6 823b921721f5 f921806c5346 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi131 smithi131 running (8m) 104s ago 18m 56.0M 2048M 16.2.14-548-g81bd20d6 823b921721f5 83bc30cf7493 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:mon.smithi196 smithi196 running (7m) 2m ago 16m 42.7M 2048M 16.2.14-548-g81bd20d6 823b921721f5 28e82f762666 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.0.1.smithi131.qccypf smithi131 *:2049 running (3m) 104s ago 9m 135M - 3.5 823b921721f5 b99130858f09 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:nfs.foo.1.1.smithi196.rqnuju smithi196 *:2049 running (2m) 2m ago 9m 62.0M - 3.5 823b921721f5 faf5deba11ee 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi131 smithi131 *:9100 running (2m) 104s ago 16m 12.2M - 1.3.1 1dbe0e931976 9edb16b58848 2024-02-19T22:02:47.953 INFO:teuthology.orchestra.run.smithi131.stdout:node-exporter.smithi196 smithi196 *:9100 running (2m) 2m ago 15m 3632k - 1.3.1 1dbe0e931976 b49f05a21958 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:osd.0 smithi131 running (7m) 104s ago 15m 123M 1944M 16.2.14-548-g81bd20d6 823b921721f5 a0e5546e07ca 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:osd.1 smithi131 running (6m) 104s ago 15m 113M 1944M 16.2.14-548-g81bd20d6 823b921721f5 f8444cfee8d5 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:osd.2 smithi131 running (6m) 104s ago 14m 92.4M 1944M 16.2.14-548-g81bd20d6 823b921721f5 433a290cc652 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:osd.3 smithi131 running (5m) 104s ago 14m 121M 1944M 16.2.14-548-g81bd20d6 823b921721f5 840766dde496 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:osd.4 smithi196 running (5m) 2m ago 14m 128M 2712M 16.2.14-548-g81bd20d6 823b921721f5 209ac9ce00bd 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:osd.5 smithi196 running (4m) 2m ago 13m 119M 2712M 16.2.14-548-g81bd20d6 823b921721f5 4c4b41297954 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:osd.6 smithi196 running (4m) 2m ago 13m 113M 2712M 16.2.14-548-g81bd20d6 823b921721f5 e9be2f442ef3 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:osd.7 smithi196 running (3m) 2m ago 13m 138M 2712M 16.2.14-548-g81bd20d6 823b921721f5 10d1c2418ed8 2024-02-19T22:02:47.954 INFO:teuthology.orchestra.run.smithi131.stdout:prometheus.smithi131 smithi131 *:9095 running (2m) 104s ago 16m 62.7M - 2.33.4 514e6a882f6e 64ccabaf7590 2024-02-19T22:02:48.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:46.648098+0000 mgr.smithi131.cjubsp (mgr.14720) 413 : cluster [DBG] pgmap v297: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:48.642 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph versions' 2024-02-19T22:02:49.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:48 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:47.944100+0000 mgr.smithi131.cjubsp (mgr.14720) 414 : audit [DBG] from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:02:49.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:47.944100+0000 mgr.smithi131.cjubsp (mgr.14720) 414 : audit [DBG] from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:02:50.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:49 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:48.648558+0000 mgr.smithi131.cjubsp (mgr.14720) 415 : cluster [DBG] pgmap v298: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:50.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:48.648558+0000 mgr.smithi131.cjubsp (mgr.14720) 415 : cluster [DBG] pgmap v298: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:50.796 INFO:teuthology.orchestra.run.smithi131.stdout:{ 2024-02-19T22:02:50.796 INFO:teuthology.orchestra.run.smithi131.stdout: "mon": { 2024-02-19T22:02:50.796 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: "mgr": { 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: "osd": { 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 8 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: "mds": { 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 2 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: "overall": { 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)": 14 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T22:02:50.797 INFO:teuthology.orchestra.run.smithi131.stdout:} 2024-02-19T22:02:51.498 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-02-19T22:02:51.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:50.649048+0000 mgr.smithi131.cjubsp (mgr.14720) 416 : cluster [DBG] pgmap v299: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:51.756 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:51 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:50.797417+0000 mon.smithi131 (mon.0) 924 : audit [DBG] from='client.? 172.21.15.131:0/1340043657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:02:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:50.649048+0000 mgr.smithi131.cjubsp (mgr.14720) 416 : cluster [DBG] pgmap v299: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:52.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:50.797417+0000 mon.smithi131 (mon.0) 924 : audit [DBG] from='client.? 172.21.15.131:0/1340043657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:02:53.707 INFO:teuthology.orchestra.run.smithi131.stdout:true 2024-02-19T22:02:54.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:52.649591+0000 mgr.smithi131.cjubsp (mgr.14720) 417 : cluster [DBG] pgmap v300: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:54.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:53 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:53.696043+0000 mon.smithi131 (mon.0) 925 : audit [DBG] from='client.? 172.21.15.131:0/2598814004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:02:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:52.649591+0000 mgr.smithi131.cjubsp (mgr.14720) 417 : cluster [DBG] pgmap v300: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:54.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:53.696043+0000 mon.smithi131 (mon.0) 925 : audit [DBG] from='client.? 172.21.15.131:0/2598814004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:02:54.427 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -e sha1=81bd20d634209c7cb82c18be12b4b5a05643ebf1 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-02-19T22:02:55.863 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:55 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:54.650055+0000 mgr.smithi131.cjubsp (mgr.14720) 418 : cluster [DBG] pgmap v301: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:56.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:54.650055+0000 mgr.smithi131.cjubsp (mgr.14720) 418 : cluster [DBG] pgmap v301: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:02:56.589 INFO:teuthology.orchestra.run.smithi131.stdout: "ceph version 16.2.14-548-g81bd20d6 (81bd20d634209c7cb82c18be12b4b5a05643ebf1) pacific (stable)" 2024-02-19T22:02:56.915 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:56 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:56.578091+0000 mon.smithi131 (mon.0) 926 : audit [DBG] from='client.? 172.21.15.131:0/2713059603' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:02:57.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:56.578091+0000 mon.smithi131 (mon.0) 926 : audit [DBG] from='client.? 172.21.15.131:0/2713059603' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-19T22:02:57.241 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-02-19T22:02:57.242 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-02-19T22:02:57.242 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- ceph orch ls -f json 2024-02-19T22:02:57.968 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:57 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:56.650532+0000 mgr.smithi131.cjubsp (mgr.14720) 419 : cluster [DBG] pgmap v302: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:58.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:56.650532+0000 mgr.smithi131.cjubsp (mgr.14720) 419 : cluster [DBG] pgmap v302: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:02:59.371 INFO:teuthology.orchestra.run.smithi131.stdout: 2024-02-19T22:02:59.371 INFO:teuthology.orchestra.run.smithi131.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-02-19T21:44:53.589413Z", "last_refresh": "2024-02-19T22:01:03.599138Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-02-19T21:44:40.820307Z", "last_refresh": "2024-02-19T22:00:09.508054Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-02-19T21:44:50.364803Z", "last_refresh": "2024-02-19T22:01:03.599282Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-02-19T21:51:14.850696Z", "last_refresh": "2024-02-19T22:00:09.508671Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-02-19T21:44:39.551937Z", "last_refresh": "2024-02-19T22:00:09.508207Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi131:172.21.15.131=smithi131", "smithi196:172.21.15.196=smithi196"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-02-19T21:45:52.401306Z", "last_refresh": "2024-02-19T22:00:09.508286Z", "running": 2, "size": 2}}, {"events": ["2024-02-19T21:54:27.969771Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "status": {"created": "2024-02-19T21:53:36.350513Z", "last_refresh": "2024-02-19T22:00:09.508731Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-02-19T21:44:51.697757Z", "last_refresh": "2024-02-19T22:00:09.508352Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "823b921721f579487e02cd293080702537bcba9f4f96b9041a06d7563de513aa", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:7fa249cc0191869e2bcc743303b0da933d04d152b2397edb19ec992e48553ab9", "last_refresh": "2024-02-19T22:00:09.508431Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-02-19T21:44:49.047885Z", "last_refresh": "2024-02-19T22:01:03.599401Z", "ports": [9095], "running": 1, "size": 1}}] 2024-02-19T22:03:00.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:02:59 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:02:58.651035+0000 mgr.smithi131.cjubsp (mgr.14720) 420 : cluster [DBG] pgmap v303: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:00.124 INFO:tasks.cephadm:nfs.foo has 2/2 2024-02-19T22:03:00.124 DEBUG:teuthology.parallel:result is None 2024-02-19T22:03:00.124 INFO:teuthology.run_tasks:Running task vip.exec... 2024-02-19T22:03:00.134 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T22:03:00.134 DEBUG:teuthology.orchestra.run.smithi131:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-02-19T22:03:00.166 INFO:teuthology.orchestra.run.smithi131.stderr:+ umount /mnt/foo 2024-02-19T22:03:00.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:02:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:02:58.651035+0000 mgr.smithi131.cjubsp (mgr.14720) 420 : cluster [DBG] pgmap v303: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:00.196 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-19T22:03:00.205 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T22:03:00.206 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-02-19T22:03:00.933 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:00 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:02:59.367855+0000 mgr.smithi131.cjubsp (mgr.14720) 421 : audit [DBG] from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T22:03:01.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:02:59.367855+0000 mgr.smithi131.cjubsp (mgr.14720) 421 : audit [DBG] from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-19T22:03:01.932 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:01 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:03:00.651540+0000 mgr.smithi131.cjubsp (mgr.14720) 422 : cluster [DBG] pgmap v304: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:02.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:03:00.651540+0000 mgr.smithi131.cjubsp (mgr.14720) 422 : cluster [DBG] pgmap v304: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:02.403 INFO:teuthology.orchestra.run.smithi131.stdout:foo 2024-02-19T22:03:03.127 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph nfs export ls foo --detailed' 2024-02-19T22:03:03.857 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:03:02.389522+0000 mgr.smithi131.cjubsp (mgr.14720) 423 : audit [DBG] from='client.34458 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:03:03.857 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:03 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:03:02.652009+0000 mgr.smithi131.cjubsp (mgr.14720) 424 : cluster [DBG] pgmap v305: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:03:02.389522+0000 mgr.smithi131.cjubsp (mgr.14720) 423 : audit [DBG] from='client.34458 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:03:04.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:03:02.652009+0000 mgr.smithi131.cjubsp (mgr.14720) 424 : cluster [DBG] pgmap v305: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:03:04.046621+0000 mon.smithi131 (mon.0) 927 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:03:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:03:04.337924+0000 mon.smithi131 (mon.0) 928 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:03:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:03:04.338510+0000 mon.smithi131 (mon.0) 929 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:03:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:03:04.342406+0000 mon.smithi131 (mon.0) 930 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:03:05.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:04 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:03:04.347007+0000 mon.smithi131 (mon.0) 931 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:03:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:03:04.046621+0000 mon.smithi131 (mon.0) 927 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-19T22:03:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:03:04.337924+0000 mon.smithi131 (mon.0) 928 : audit [DBG] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-19T22:03:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:03:04.338510+0000 mon.smithi131 (mon.0) 929 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-19T22:03:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:03:04.342406+0000 mon.smithi131 (mon.0) 930 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:03:05.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:03:04.347007+0000 mon.smithi131 (mon.0) 931 : audit [INF] from='mgr.14720 172.21.15.131:0/3179867039' entity='mgr.smithi131.cjubsp' 2024-02-19T22:03:05.555 INFO:teuthology.orchestra.run.smithi131.stdout:[ 2024-02-19T22:03:05.555 INFO:teuthology.orchestra.run.smithi131.stdout: { 2024-02-19T22:03:05.555 INFO:teuthology.orchestra.run.smithi131.stdout: "export_id": 1, 2024-02-19T22:03:05.555 INFO:teuthology.orchestra.run.smithi131.stdout: "path": "/", 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "cluster_id": "foo", 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "pseudo": "/fake", 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "access_type": "RW", 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "squash": "no_root_squash", 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "security_label": true, 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "protocols": [ 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: 4 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "transports": [ 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "TCP" 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: ], 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "fsal": { 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "name": "CEPH", 2024-02-19T22:03:05.556 INFO:teuthology.orchestra.run.smithi131.stdout: "user_id": "nfs.foo.1", 2024-02-19T22:03:05.557 INFO:teuthology.orchestra.run.smithi131.stdout: "fs_name": "foofs" 2024-02-19T22:03:05.557 INFO:teuthology.orchestra.run.smithi131.stdout: }, 2024-02-19T22:03:05.557 INFO:teuthology.orchestra.run.smithi131.stdout: "clients": [] 2024-02-19T22:03:05.557 INFO:teuthology.orchestra.run.smithi131.stdout: } 2024-02-19T22:03:05.557 INFO:teuthology.orchestra.run.smithi131.stdout:] 2024-02-19T22:03:05.885 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:05 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:03:04.652484+0000 mgr.smithi131.cjubsp (mgr.14720) 425 : cluster [DBG] pgmap v306: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:03:06.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:03:04.652484+0000 mgr.smithi131.cjubsp (mgr.14720) 425 : cluster [DBG] pgmap v306: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-19T22:03:06.225 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'rados -p .nfs --all ls -' 2024-02-19T22:03:06.980 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:06 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:03:05.527908+0000 mgr.smithi131.cjubsp (mgr.14720) 426 : audit [DBG] from='client.34462 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:03:07.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:03:05.527908+0000 mgr.smithi131.cjubsp (mgr.14720) 426 : audit [DBG] from='client.34462 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-19T22:03:08.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:03:06.652962+0000 mgr.smithi131.cjubsp (mgr.14720) 427 : cluster [DBG] pgmap v307: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:08.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:07 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:03:06.769066+0000 mon.smithi131 (mon.0) 932 : cluster [DBG] mgrmap e35: smithi131.cjubsp(active, since 8m), standbys: smithi196.voprex 2024-02-19T22:03:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:03:06.652962+0000 mgr.smithi131.cjubsp (mgr.14720) 427 : cluster [DBG] pgmap v307: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:08.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:03:06.769066+0000 mon.smithi131 (mon.0) 932 : cluster [DBG] mgrmap e35: smithi131.cjubsp(active, since 8m), standbys: smithi196.voprex 2024-02-19T22:03:08.295 INFO:teuthology.orchestra.run.smithi131.stdout:foo grace 2024-02-19T22:03:08.295 INFO:teuthology.orchestra.run.smithi131.stdout:foo rec-0000000000000005:nfs.foo.1 2024-02-19T22:03:08.295 INFO:teuthology.orchestra.run.smithi131.stdout:foo rec-0000000000000005:nfs.foo.0 2024-02-19T22:03:08.295 INFO:teuthology.orchestra.run.smithi131.stdout:foo conf-nfs.foo 2024-02-19T22:03:08.295 INFO:teuthology.orchestra.run.smithi131.stdout:foo export-1 2024-02-19T22:03:08.925 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 -- bash -c 'ceph config get mgr mgr/cephadm/migration_current | grep 5' 2024-02-19T22:03:10.005 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:09 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:03:08.653554+0000 mgr.smithi131.cjubsp (mgr.14720) 428 : cluster [DBG] pgmap v308: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:10.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:03:08.653554+0000 mgr.smithi131.cjubsp (mgr.14720) 428 : cluster [DBG] pgmap v308: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-19T22:03:11.023 INFO:teuthology.orchestra.run.smithi131.stdout:5 2024-02-19T22:03:11.664 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-02-19T22:03:11.675 INFO:tasks.cephadm:Teardown begin 2024-02-19T22:03:11.675 DEBUG:teuthology.orchestra.run.smithi131:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-19T22:03:11.707 DEBUG:teuthology.orchestra.run.smithi196:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-19T22:03:11.745 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-02-19T22:03:11.745 DEBUG:teuthology.orchestra.run.smithi131:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-02-19T22:03:11.765 DEBUG:teuthology.orchestra.run.smithi196:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-02-19T22:03:11.806 INFO:tasks.cephadm:Stopping all daemons... 2024-02-19T22:03:11.806 INFO:tasks.cephadm.mon.smithi131:Stopping mon.smithi131... 2024-02-19T22:03:11.806 DEBUG:teuthology.orchestra.run.smithi131:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131 2024-02-19T22:03:12.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: cluster 2024-02-19T22:03:10.654020+0000 mgr.smithi131.cjubsp (mgr.14720) 429 : cluster [DBG] pgmap v309: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-19T22:03:12.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:11 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: audit 2024-02-19T22:03:11.012476+0000 mon.smithi131 (mon.0) 933 : audit [DBG] from='client.? 172.21.15.131:0/2269909941' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-02-19T22:03:12.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:11 smithi131 systemd[1]: Stopping Ceph mon.smithi131 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:03:12.006 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:11 smithi131 bash[100818]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi131" found: no such container 2024-02-19T22:03:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: cluster 2024-02-19T22:03:10.654020+0000 mgr.smithi131.cjubsp (mgr.14720) 429 : cluster [DBG] pgmap v309: 129 pgs: 129 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-19T22:03:12.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: audit 2024-02-19T22:03:11.012476+0000 mon.smithi131 (mon.0) 933 : audit [DBG] from='client.? 172.21.15.131:0/2269909941' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-02-19T22:03:12.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T22:03:12.115+0000 7fbb10fdf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi131 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-02-19T22:03:12.506 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:12 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131[76781]: debug 2024-02-19T22:03:12.115+0000 7fbb10fdf700 -1 mon.smithi131@0(leader) e2 *** Got Signal Terminated *** 2024-02-19T22:03:12.992 INFO:journalctl@ceph.mon.smithi131.smithi131.stdout:Feb 19 22:03:12 smithi131 bash[100858]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi131 2024-02-19T22:03:12.999 DEBUG:teuthology.orchestra.run.smithi131:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi131.service' 2024-02-19T22:03:13.053 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:03:13.054 INFO:tasks.cephadm.mon.smithi131:Stopped mon.smithi131 2024-02-19T22:03:13.054 INFO:tasks.cephadm.mon.smithi196:Stopping mon.smithi196... 2024-02-19T22:03:13.054 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi196 2024-02-19T22:03:13.363 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:13 smithi196 systemd[1]: Stopping Ceph mon.smithi196 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:03:13.364 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:13 smithi196 bash[69658]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon.smithi196" found: no such container 2024-02-19T22:03:13.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T22:03:13.363+0000 7ff50d502700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi196 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-02-19T22:03:13.673 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196[57368]: debug 2024-02-19T22:03:13.363+0000 7ff50d502700 -1 mon.smithi196@1(peon) e2 *** Got Signal Terminated *** 2024-02-19T22:03:14.173 INFO:journalctl@ceph.mon.smithi196.smithi196.stdout:Feb 19 22:03:13 smithi196 bash[69696]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-mon-smithi196 2024-02-19T22:03:14.349 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@mon.smithi196.service' 2024-02-19T22:03:14.392 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:03:14.392 INFO:tasks.cephadm.mon.smithi196:Stopped mon.smithi196 2024-02-19T22:03:14.392 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-02-19T22:03:14.392 DEBUG:teuthology.orchestra.run.smithi131:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.0 2024-02-19T22:03:14.686 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 22:03:14 smithi131 systemd[1]: Stopping Ceph osd.0 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:03:14.686 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 22:03:14 smithi131 bash[100988]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.0" found: no such container 2024-02-19T22:03:15.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 22:03:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-0[80308]: debug 2024-02-19T22:03:14.685+0000 7f67d4fcb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T22:03:15.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 22:03:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-0[80308]: debug 2024-02-19T22:03:14.685+0000 7f67d4fcb700 -1 osd.0 111 *** Got signal Terminated *** 2024-02-19T22:03:15.006 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 22:03:14 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-0[80308]: debug 2024-02-19T22:03:14.685+0000 7f67d4fcb700 -1 osd.0 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T22:03:20.506 INFO:journalctl@ceph.osd.0.smithi131.stdout:Feb 19 22:03:20 smithi131 bash[101028]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-0 2024-02-19T22:03:22.058 DEBUG:teuthology.orchestra.run.smithi131:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.0.service' 2024-02-19T22:03:22.102 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:03:22.102 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-02-19T22:03:22.102 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-02-19T22:03:22.102 DEBUG:teuthology.orchestra.run.smithi131:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.1 2024-02-19T22:03:22.424 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 22:03:22 smithi131 systemd[1]: Stopping Ceph osd.1 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:03:22.424 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 22:03:22 smithi131 bash[101267]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.1" found: no such container 2024-02-19T22:03:22.755 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 22:03:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-1[82812]: debug 2024-02-19T22:03:22.424+0000 7f9da064e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T22:03:22.756 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 22:03:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-1[82812]: debug 2024-02-19T22:03:22.424+0000 7f9da064e700 -1 osd.1 111 *** Got signal Terminated *** 2024-02-19T22:03:22.756 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 22:03:22 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-1[82812]: debug 2024-02-19T22:03:22.424+0000 7f9da064e700 -1 osd.1 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T22:03:28.256 INFO:journalctl@ceph.osd.1.smithi131.stdout:Feb 19 22:03:27 smithi131 bash[101306]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-1 2024-02-19T22:03:29.637 DEBUG:teuthology.orchestra.run.smithi131:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.1.service' 2024-02-19T22:03:29.682 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:03:29.682 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-02-19T22:03:29.682 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-02-19T22:03:29.682 DEBUG:teuthology.orchestra.run.smithi131:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.2 2024-02-19T22:03:29.990 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 22:03:29 smithi131 systemd[1]: Stopping Ceph osd.2 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:03:29.990 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 22:03:29 smithi131 bash[101547]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.2" found: no such container 2024-02-19T22:03:30.255 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 22:03:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-2[85278]: debug 2024-02-19T22:03:29.990+0000 7f38d5faf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T22:03:30.256 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 22:03:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-2[85278]: debug 2024-02-19T22:03:29.990+0000 7f38d5faf700 -1 osd.2 111 *** Got signal Terminated *** 2024-02-19T22:03:30.256 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 22:03:29 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-2[85278]: debug 2024-02-19T22:03:29.990+0000 7f38d5faf700 -1 osd.2 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T22:03:36.005 INFO:journalctl@ceph.osd.2.smithi131.stdout:Feb 19 22:03:35 smithi131 bash[101585]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-2 2024-02-19T22:03:37.355 DEBUG:teuthology.orchestra.run.smithi131:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.2.service' 2024-02-19T22:03:37.396 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:03:37.397 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-02-19T22:03:37.397 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-02-19T22:03:37.397 DEBUG:teuthology.orchestra.run.smithi131:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.3 2024-02-19T22:03:37.706 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 22:03:37 smithi131 systemd[1]: Stopping Ceph osd.3 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:03:37.706 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 22:03:37 smithi131 bash[101824]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.3" found: no such container 2024-02-19T22:03:38.006 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 22:03:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-3[87743]: debug 2024-02-19T22:03:37.705+0000 7fe8d9404700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T22:03:38.006 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 22:03:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-3[87743]: debug 2024-02-19T22:03:37.705+0000 7fe8d9404700 -1 osd.3 111 *** Got signal Terminated *** 2024-02-19T22:03:38.006 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 22:03:37 smithi131 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-3[87743]: debug 2024-02-19T22:03:37.705+0000 7fe8d9404700 -1 osd.3 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T22:03:40.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:40 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:40.531+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:41.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:41 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:41.570+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:42.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:42.100+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:42.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:42.574+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:42.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:42 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:42.822+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:43.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:43.096+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:43.505 INFO:journalctl@ceph.osd.3.smithi131.stdout:Feb 19 22:03:43 smithi131 bash[101865]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-3 2024-02-19T22:03:43.831 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:43.567+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:44.143 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:43 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:43.830+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:44.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:44.144+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:44.849 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:44.594+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:44.849 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:44.746+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:44.915 DEBUG:teuthology.orchestra.run.smithi131:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.3.service' 2024-02-19T22:03:44.957 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:03:44.957 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-02-19T22:03:44.957 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-02-19T22:03:44.957 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.4 2024-02-19T22:03:45.160 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:44 smithi196 systemd[1]: Stopping Ceph osd.4 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:03:45.160 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:45 smithi196 bash[69818]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.4" found: no such container 2024-02-19T22:03:45.160 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:44 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:44.851+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:45.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:45.159+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:45.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:45.288+0000 7fc5781b0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T22:03:45.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:45.288+0000 7fc5781b0700 -1 osd.4 111 *** Got signal Terminated *** 2024-02-19T22:03:45.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:45.288+0000 7fc5781b0700 -1 osd.4 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T22:03:45.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:45.808+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:45.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:45.643+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:45.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:45 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:45.739+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:46.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:46.147+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:46.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:46.794+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:46.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:46.646+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:46.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:46 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:46.753+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:47.632 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:47.185+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:47.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:47.749+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:47.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:47.718+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:47.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:47 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:47.633+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:48.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:48.172+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:48.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:48.742+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:48.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:48.742+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:48.924 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:48.725+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:48.924 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:48.725+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:48.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:48.587+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:48.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:48 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:48.587+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:49.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:49.169+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:49.423 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:49.169+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:24.576218+0000 front 2024-02-19T22:03:24.576093+0000 (oldest deadline 2024-02-19T22:03:48.675965+0000) 2024-02-19T22:03:49.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:49.732+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:49.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:49.732+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:49.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:49.768+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:49.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:49.768+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:49.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:49.627+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:49.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:49 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:49.627+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:50.606 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:50.214+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.274814+0000 front 2024-02-19T22:03:18.274737+0000 (oldest deadline 2024-02-19T22:03:41.774609+0000) 2024-02-19T22:03:50.606 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4[59983]: debug 2024-02-19T22:03:50.214+0000 7fc57512d700 -1 osd.4 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:24.576218+0000 front 2024-02-19T22:03:24.576093+0000 (oldest deadline 2024-02-19T22:03:48.675965+0000) 2024-02-19T22:03:50.881 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:50.735+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:50.881 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:50.735+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:50.881 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:50.606+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:50.881 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:50.606+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:50.882 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:50.771+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:50.882 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:50 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:50.771+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:51.173 INFO:journalctl@ceph.osd.4.smithi196.stdout:Feb 19 22:03:50 smithi196 bash[69858]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-4 2024-02-19T22:03:51.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:51.780+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:51.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:51.780+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:51.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:51.590+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:51.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:51.591+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:51.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:51.749+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:51.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:51 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:51.749+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:52.775 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.4.service' 2024-02-19T22:03:52.819 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:03:52.819 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-02-19T22:03:52.819 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-02-19T22:03:52.819 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.5 2024-02-19T22:03:52.884 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:52.801+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:52.885 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:52.801+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:52.885 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:52.590+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:52.885 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:52.590+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:52.885 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:52.701+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:52.885 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:52 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:52.701+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:53.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:52 smithi196 systemd[1]: Stopping Ceph osd.5 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:03:53.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:52 smithi196 bash[70104]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.5" found: no such container 2024-02-19T22:03:53.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:53.124+0000 7fe439457700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T22:03:53.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:53.124+0000 7fe439457700 -1 osd.5 111 *** Got signal Terminated *** 2024-02-19T22:03:53.173 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:53.124+0000 7fe439457700 -1 osd.5 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T22:03:53.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:53.684+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:53.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:53.684+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:53.924 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:53.769+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:53.924 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:53.769+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:53.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:53.612+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:53.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:53 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:53.612+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:54.848 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:54.695+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:54.848 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:54.695+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:54.849 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:54.583+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:54.849 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:54.583+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:54.849 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:54.765+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:54.849 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:54 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:54.765+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:55.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:55.668+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:55.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:55.668+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:55.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:55.788+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:55.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:55.788+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:55.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:55.583+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:55.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:55.583+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:55.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:55 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:55.583+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:03:56.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:56.659+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:56.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:56.659+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:56.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:56.769+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:56.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:56.769+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:56.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:56.627+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:56.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:56.627+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:56.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:56 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:56.627+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:03:57.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:57.803+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:17.890840+0000 front 2024-02-19T22:03:17.890762+0000 (oldest deadline 2024-02-19T22:03:42.590693+0000) 2024-02-19T22:03:57.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5[61933]: debug 2024-02-19T22:03:57.803+0000 7fe436bd5700 -1 osd.5 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:22.591182+0000 front 2024-02-19T22:03:22.591195+0000 (oldest deadline 2024-02-19T22:03:47.891096+0000) 2024-02-19T22:03:57.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:57.629+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:57.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:57.629+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:57.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:57.629+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:03:57.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:57.613+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:57.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:57.613+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:57.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:57 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:57.613+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:03:58.923 INFO:journalctl@ceph.osd.5.smithi196.stdout:Feb 19 22:03:58 smithi196 bash[70143]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-5 2024-02-19T22:03:58.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:58.636+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:58.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:58.636+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:58.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:58.636+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:03:58.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:58.623+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:58.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:58.623+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:58.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:58 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:58.623+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:03:59.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:59.672+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:03:59.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:59.672+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:03:59.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:03:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:03:59.672+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:03:59.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:59.622+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:03:59.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:59.622+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:03:59.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:03:59 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:03:59.622+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:04:00.566 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.5.service' 2024-02-19T22:04:00.650 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:04:00.650 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-02-19T22:04:00.650 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-02-19T22:04:00.650 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.6 2024-02-19T22:04:00.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:00.624+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:04:00.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:00.624+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:04:00.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:00.624+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:04:00.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:00 smithi196 systemd[1]: Stopping Ceph osd.6 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:04:00.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:00 smithi196 bash[70381]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.6" found: no such container 2024-02-19T22:04:00.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:00.655+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:00.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:00.655+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:00.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:00 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:00.655+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:01.423 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:01.028+0000 7efc3de78700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T22:04:01.423 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:01.028+0000 7efc3de78700 -1 osd.6 111 *** Got signal Terminated *** 2024-02-19T22:04:01.423 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:01.028+0000 7efc3de78700 -1 osd.6 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T22:04:01.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:01.616+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:04:01.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:01.616+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:04:01.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:01.616+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:04:01.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:01.631+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:01.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:01.631+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:01.924 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:01 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:01.631+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:02.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:02.650+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:04:02.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:02.650+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:04:02.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:02.650+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:04:02.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:02.599+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:02.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:02.599+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:02.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:02 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:02.599+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:03.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:03.696+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:04:03.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:03.696+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:04:03.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:03.696+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:04:03.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:03.607+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:03.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:03.607+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:03.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:03 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:03.607+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:04.848 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:04.647+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:04:04.848 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:04.647+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:04:04.848 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:04.647+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:04:04.848 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:04.562+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:04.848 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:04.562+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:04.849 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:04 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:04.562+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:05.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:05.543+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:05.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:05.543+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:05.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:05.543+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:05.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:05.543+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:05.923 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:05.679+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:16.881534+0000 front 2024-02-19T22:03:16.881314+0000 (oldest deadline 2024-02-19T22:03:39.781325+0000) 2024-02-19T22:04:05.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:05.679+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.282459+0000 front 2024-02-19T22:03:23.282506+0000 (oldest deadline 2024-02-19T22:03:48.582395+0000) 2024-02-19T22:04:05.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:05.679+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.383618+0000 front 2024-02-19T22:03:34.383726+0000 (oldest deadline 2024-02-19T22:03:55.483651+0000) 2024-02-19T22:04:05.924 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:05 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6[63822]: debug 2024-02-19T22:04:05.679+0000 7efc395f2700 -1 osd.6 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:41.185271+0000 front 2024-02-19T22:03:41.185225+0000 (oldest deadline 2024-02-19T22:04:05.285123+0000) 2024-02-19T22:04:06.673 INFO:journalctl@ceph.osd.6.smithi196.stdout:Feb 19 22:04:06 smithi196 bash[70421]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-6 2024-02-19T22:04:06.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:06.511+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:06.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:06.511+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:06.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:06.511+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:06.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:06 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:06.511+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:07.860 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:07.543+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:07.860 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:07.543+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:07.860 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:07.543+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:07.861 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:07 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:07.543+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:08.221 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.6.service' 2024-02-19T22:04:08.263 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:04:08.264 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-02-19T22:04:08.264 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-02-19T22:04:08.264 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.7 2024-02-19T22:04:08.500 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 systemd[1]: Stopping Ceph osd.7 for e8e4a71a-cf6f-11ee-95bb-87774f69a715... 2024-02-19T22:04:08.500 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 bash[70661]: Error: no container with name or ID "ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd.7" found: no such container 2024-02-19T22:04:08.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:08.499+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:08.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:08.499+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:08.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:08.499+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:08.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:08.499+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:08.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:08.595+0000 7ff664102700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-02-19T22:04:08.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:08.595+0000 7ff664102700 -1 osd.7 111 *** Got signal Terminated *** 2024-02-19T22:04:08.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:08 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:08.595+0000 7ff664102700 -1 osd.7 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-19T22:04:09.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:09.516+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:09.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:09.516+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:09.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:09.516+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:09.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:09 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:09.516+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:10.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:10.471+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:10.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:10.471+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:10.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:10.471+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:10.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:10 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:10.471+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:11.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:11.432+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:11.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:11.432+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:11.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:11.432+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:11.923 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:11 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:11.432+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:12.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:12.390+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:12.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:12.390+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:12.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:12.390+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:12.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:12 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:12.390+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:13.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:13.402+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6806 osd.0 since back 2024-02-19T22:03:18.549533+0000 front 2024-02-19T22:03:18.549710+0000 (oldest deadline 2024-02-19T22:03:43.849608+0000) 2024-02-19T22:04:13.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:13.402+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6814 osd.1 since back 2024-02-19T22:03:23.850014+0000 front 2024-02-19T22:03:23.850079+0000 (oldest deadline 2024-02-19T22:03:48.550118+0000) 2024-02-19T22:04:13.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:13.402+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6822 osd.2 since back 2024-02-19T22:03:34.351396+0000 front 2024-02-19T22:03:34.351343+0000 (oldest deadline 2024-02-19T22:03:57.251246+0000) 2024-02-19T22:04:13.673 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:13 smithi196 ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7[65716]: debug 2024-02-19T22:04:13.402+0000 7ff66007d700 -1 osd.7 111 heartbeat_check: no reply from 172.21.15.131:6830 osd.3 since back 2024-02-19T22:03:39.953133+0000 front 2024-02-19T22:03:39.953114+0000 (oldest deadline 2024-02-19T22:04:05.253011+0000) 2024-02-19T22:04:14.423 INFO:journalctl@ceph.osd.7.smithi196.stdout:Feb 19 22:04:14 smithi196 bash[70700]: ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715-osd-7 2024-02-19T22:04:15.838 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e8e4a71a-cf6f-11ee-95bb-87774f69a715@osd.7.service' 2024-02-19T22:04:15.879 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-19T22:04:15.879 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-02-19T22:04:15.880 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 --force --keep-logs 2024-02-19T22:07:11.611 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 --force --keep-logs 2024-02-19T22:07:31.193 DEBUG:teuthology.orchestra.run.smithi131:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-19T22:07:31.231 DEBUG:teuthology.orchestra.run.smithi196:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-19T22:07:31.262 INFO:tasks.cephadm:Archiving crash dumps... 2024-02-19T22:07:31.264 DEBUG:teuthology.misc:Transferring archived files from smithi131:/var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/crash to /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773/remote/ubuntu@smithi131.front.sepia.ceph.com/crash 2024-02-19T22:07:31.265 DEBUG:teuthology.orchestra.run.smithi131:> sudo tar c -f - -C /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/crash -- . 2024-02-19T22:07:31.309 INFO:teuthology.orchestra.run.smithi131.stderr:tar: /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/crash: Cannot open: No such file or directory 2024-02-19T22:07:31.309 INFO:teuthology.orchestra.run.smithi131.stderr:tar: Error is not recoverable: exiting now 2024-02-19T22:07:31.311 DEBUG:teuthology.misc:Transferring archived files from smithi196:/var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/crash to /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773/remote/ubuntu@smithi196.front.sepia.ceph.com/crash 2024-02-19T22:07:31.312 DEBUG:teuthology.orchestra.run.smithi196:> sudo tar c -f - -C /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/crash -- . 2024-02-19T22:07:31.356 INFO:teuthology.orchestra.run.smithi196.stderr:tar: /var/lib/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/crash: Cannot open: No such file or directory 2024-02-19T22:07:31.356 INFO:teuthology.orchestra.run.smithi196.stderr:tar: Error is not recoverable: exiting now 2024-02-19T22:07:31.357 INFO:tasks.cephadm:Checking cluster log for badness... 2024-02-19T22:07:31.357 DEBUG:teuthology.orchestra.run.smithi131:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-19T22:07:31.389 INFO:teuthology.orchestra.run.smithi131.stdout:2024-02-19T21:48:05.042519+0000 mon.smithi131 (mon.0) 350 : cluster [WRN] Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-02-19T22:07:31.390 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-02-19T22:07:31.390 DEBUG:teuthology.orchestra.run.smithi131:> sudo egrep '\[SEC\]' /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-19T22:07:31.460 DEBUG:teuthology.orchestra.run.smithi131:> sudo egrep '\[ERR\]' /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-19T22:07:31.530 DEBUG:teuthology.orchestra.run.smithi131:> sudo egrep '\[WRN\]' /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-19T22:07:31.599 INFO:teuthology.orchestra.run.smithi131.stdout:2024-02-19T21:48:05.042519+0000 mon.smithi131 (mon.0) 350 : cluster [WRN] Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-02-19T22:07:31.600 INFO:tasks.cephadm:Compressing logs... 2024-02-19T22:07:31.600 DEBUG:teuthology.orchestra.run.smithi131:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-02-19T22:07:31.643 DEBUG:teuthology.orchestra.run.smithi196:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-02-19T22:07:31.669 INFO:teuthology.orchestra.run.smithi131.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-02-19T22:07:31.669 INFO:teuthology.orchestra.run.smithi131.stderr:: No such file or directory 2024-02-19T22:07:31.669 INFO:teuthology.orchestra.run.smithi131.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-volume.log 2024-02-19T22:07:31.669 INFO:teuthology.orchestra.run.smithi131.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.2.log 2024-02-19T22:07:31.670 INFO:teuthology.orchestra.run.smithi131.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.audit.log 2024-02-19T22:07:31.670 INFO:teuthology.orchestra.run.smithi131.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-volume.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.cephadm.log 2024-02-19T22:07:31.670 INFO:teuthology.orchestra.run.smithi131.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.3.log 2024-02-19T22:07:31.671 INFO:teuthology.orchestra.run.smithi131.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.cephadm.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.3.log: 84.9% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.cephadm.log.gz 2024-02-19T22:07:31.672 INFO:teuthology.orchestra.run.smithi196.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-02-19T22:07:31.672 INFO:teuthology.orchestra.run.smithi196.stderr:: No such file or directory 2024-02-19T22:07:31.672 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.6.log 2024-02-19T22:07:31.673 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-volume.log 2024-02-19T22:07:31.674 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.audit.log 2024-02-19T22:07:31.674 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.6.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.cephadm.log 2024-02-19T22:07:31.674 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mds.foofs.smithi196.hhhkwt.log 2024-02-19T22:07:31.675 INFO:teuthology.orchestra.run.smithi131.stderr: 91.0% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-volume.log.gz 2024-02-19T22:07:31.675 INFO:teuthology.orchestra.run.smithi131.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.0.log 2024-02-19T22:07:31.676 INFO:teuthology.orchestra.run.smithi131.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mon.smithi131.log 2024-02-19T22:07:31.676 INFO:teuthology.orchestra.run.smithi131.stderr: 91.5% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.audit.log.gz 2024-02-19T22:07:31.676 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.cephadm.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mds.foofs.smithi196.hhhkwt.log: 84.7% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.cephadm.log.gz 2024-02-19T22:07:31.676 INFO:teuthology.orchestra.run.smithi196.stderr: 79.8% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mds.foofs.smithi196.hhhkwt.log.gz 2024-02-19T22:07:31.676 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mgr.smithi196.voprex.log 2024-02-19T22:07:31.677 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.7.log 2024-02-19T22:07:31.677 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mgr.smithi196.voprex.log: gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mon.smithi196.log 2024-02-19T22:07:31.678 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.7.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mon.smithi196.log: 90.3% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-volume.log.gz 2024-02-19T22:07:31.678 INFO:teuthology.orchestra.run.smithi196.stderr: 91.6% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.audit.log.gz 2024-02-19T22:07:31.678 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log 2024-02-19T22:07:31.678 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.4.log 2024-02-19T22:07:31.679 INFO:teuthology.orchestra.run.smithi131.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.0.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mon.smithi131.log: gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mds.foofs.smithi131.gemwnz.log 2024-02-19T22:07:31.679 INFO:teuthology.orchestra.run.smithi131.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mgr.smithi131.cjubsp.log 2024-02-19T22:07:31.679 INFO:teuthology.orchestra.run.smithi131.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mds.foofs.smithi131.gemwnz.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mgr.smithi131.cjubsp.log: 85.9% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mds.foofs.smithi131.gemwnz.log.gz 2024-02-19T22:07:31.679 INFO:teuthology.orchestra.run.smithi131.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log 2024-02-19T22:07:31.679 INFO:teuthology.orchestra.run.smithi131.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.1.log 2024-02-19T22:07:31.680 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.4.log: 88.1% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log.gz 2024-02-19T22:07:31.681 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.5.log 2024-02-19T22:07:31.681 INFO:teuthology.orchestra.run.smithi196.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-02-19T22:07:31.682 INFO:teuthology.orchestra.run.smithi131.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log: /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.1.log: 88.1% 90.9% -- replaced with /var/log/ceph/cephadm.log.gz -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph.log.gz 2024-02-19T22:07:31.682 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T22:07:31.723 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.5.log: 89.3% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mgr.smithi196.voprex.log.gz 2024-02-19T22:07:31.970 INFO:teuthology.orchestra.run.smithi196.stderr: 92.7% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mon.smithi196.log.gz 2024-02-19T22:07:32.050 INFO:teuthology.orchestra.run.smithi131.stderr: 90.9% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mgr.smithi131.cjubsp.log.gz 2024-02-19T22:07:32.462 INFO:teuthology.orchestra.run.smithi131.stderr: 91.6% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-mon.smithi131.log.gz 2024-02-19T22:07:35.077 INFO:teuthology.orchestra.run.smithi131.stderr: 94.3% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.2.log.gz 2024-02-19T22:07:36.746 INFO:teuthology.orchestra.run.smithi196.stderr: 94.2% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.6.log.gz 2024-02-19T22:07:37.071 INFO:teuthology.orchestra.run.smithi196.stderr: 93.9% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.4.log.gz 2024-02-19T22:07:37.671 INFO:teuthology.orchestra.run.smithi131.stderr: 94.1% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.1.log.gz 2024-02-19T22:07:37.751 INFO:teuthology.orchestra.run.smithi131.stderr: 94.0% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.0.log.gz 2024-02-19T22:07:37.842 INFO:teuthology.orchestra.run.smithi131.stderr: 94.0% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.3.log.gz 2024-02-19T22:07:37.844 INFO:teuthology.orchestra.run.smithi131.stderr: 2024-02-19T22:07:37.844 INFO:teuthology.orchestra.run.smithi131.stderr:real 0m6.185s 2024-02-19T22:07:37.845 INFO:teuthology.orchestra.run.smithi131.stderr:user 0m21.871s 2024-02-19T22:07:37.845 INFO:teuthology.orchestra.run.smithi131.stderr:sys 0m0.947s 2024-02-19T22:07:38.357 INFO:teuthology.orchestra.run.smithi196.stderr: 94.1% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.5.log.gz 2024-02-19T22:07:40.406 INFO:teuthology.orchestra.run.smithi196.stderr: 94.1% -- replaced with /var/log/ceph/e8e4a71a-cf6f-11ee-95bb-87774f69a715/ceph-osd.7.log.gz 2024-02-19T22:07:40.408 INFO:teuthology.orchestra.run.smithi196.stderr: 2024-02-19T22:07:40.408 INFO:teuthology.orchestra.run.smithi196.stderr:real 0m8.746s 2024-02-19T22:07:40.408 INFO:teuthology.orchestra.run.smithi196.stderr:user 0m25.156s 2024-02-19T22:07:40.408 INFO:teuthology.orchestra.run.smithi196.stderr:sys 0m1.067s 2024-02-19T22:07:40.408 INFO:tasks.cephadm:Archiving logs... 2024-02-19T22:07:40.408 DEBUG:teuthology.misc:Transferring archived files from smithi131:/var/log/ceph to /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773/remote/ubuntu@smithi131.front.sepia.ceph.com/log 2024-02-19T22:07:40.409 DEBUG:teuthology.orchestra.run.smithi131:> sudo tar c -f - -C /var/log/ceph -- . 2024-02-19T22:07:44.053 DEBUG:teuthology.misc:Transferring archived files from smithi196:/var/log/ceph to /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773/remote/ubuntu@smithi196.front.sepia.ceph.com/log 2024-02-19T22:07:44.054 DEBUG:teuthology.orchestra.run.smithi196:> sudo tar c -f - -C /var/log/ceph -- . 2024-02-19T22:07:47.804 INFO:tasks.cephadm:Removing cluster... 2024-02-19T22:07:47.804 DEBUG:teuthology.orchestra.run.smithi131:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 --force 2024-02-19T22:07:48.153 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e8e4a71a-cf6f-11ee-95bb-87774f69a715 --force 2024-02-19T22:07:48.514 INFO:tasks.cephadm:Removing cephadm ... 2024-02-19T22:07:48.514 DEBUG:teuthology.orchestra.run.smithi131:> rm -rf /home/ubuntu/cephtest/cephadm 2024-02-19T22:07:48.532 DEBUG:teuthology.orchestra.run.smithi196:> rm -rf /home/ubuntu/cephtest/cephadm 2024-02-19T22:07:48.550 INFO:tasks.cephadm:Teardown complete 2024-02-19T22:07:48.550 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-02-19T22:07:48.560 INFO:teuthology.task.clock:Checking final clock skew... 2024-02-19T22:07:48.560 DEBUG:teuthology.orchestra.run.smithi131:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-19T22:07:48.575 DEBUG:teuthology.orchestra.run.smithi196:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-19T22:07:48.590 INFO:teuthology.orchestra.run.smithi131.stderr:bash: ntpq: command not found 2024-02-19T22:07:48.594 INFO:teuthology.orchestra.run.smithi131.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-19T22:07:48.594 INFO:teuthology.orchestra.run.smithi131.stdout:=============================================================================== 2024-02-19T22:07:48.594 INFO:teuthology.orchestra.run.smithi131.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 58 +1297us[+1297us] +/- 87ms 2024-02-19T22:07:48.594 INFO:teuthology.orchestra.run.smithi131.stdout:^* hv02.front.sepia.ceph.com 3 6 377 60 -1382us[-1388us] +/- 56ms 2024-02-19T22:07:48.594 INFO:teuthology.orchestra.run.smithi131.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 +731us[ +724us] +/- 73ms 2024-02-19T22:07:48.594 INFO:teuthology.orchestra.run.smithi131.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T22:07:48.606 INFO:teuthology.orchestra.run.smithi196.stderr:bash: ntpq: command not found 2024-02-19T22:07:48.610 INFO:teuthology.orchestra.run.smithi196.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-19T22:07:48.610 INFO:teuthology.orchestra.run.smithi196.stdout:=============================================================================== 2024-02-19T22:07:48.610 INFO:teuthology.orchestra.run.smithi196.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 57 +1285us[+1285us] +/- 87ms 2024-02-19T22:07:48.610 INFO:teuthology.orchestra.run.smithi196.stdout:^* hv02.front.sepia.ceph.com 3 6 377 58 -1382us[-1381us] +/- 56ms 2024-02-19T22:07:48.610 INFO:teuthology.orchestra.run.smithi196.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 58 +718us[ +719us] +/- 73ms 2024-02-19T22:07:48.611 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-02-19T22:07:48.611 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-02-19T22:07:48.623 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-02-19T22:07:48.624 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-02-19T22:07:48.634 DEBUG:teuthology.orchestra.run.smithi131:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-02-19T22:07:48.697 DEBUG:teuthology.orchestra.run.smithi196:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-02-19T22:07:48.757 DEBUG:teuthology.orchestra.run.smithi131:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-19T22:07:48.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-19T22:07:48.786 DEBUG:teuthology.orchestra.run.smithi196:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-19T22:07:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-19T22:07:48.815 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-02-19T22:07:48.825 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-02-19T22:07:48.835 INFO:teuthology.task.internal:Duration was 1845.889153 seconds 2024-02-19T22:07:48.835 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-02-19T22:07:48.844 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-02-19T22:07:48.845 DEBUG:teuthology.orchestra.run.smithi131:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-02-19T22:07:48.847 DEBUG:teuthology.orchestra.run.smithi196:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-02-19T22:07:48.890 INFO:teuthology.orchestra.run.smithi131.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-19T22:07:48.901 INFO:teuthology.orchestra.run.smithi196.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-19T22:07:49.186 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-02-19T22:07:49.186 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi131.front.sepia.ceph.com 2024-02-19T22:07:49.186 DEBUG:teuthology.orchestra.run.smithi131:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-02-19T22:07:49.209 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi196.front.sepia.ceph.com 2024-02-19T22:07:49.209 DEBUG:teuthology.orchestra.run.smithi196:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-02-19T22:07:49.231 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-02-19T22:07:49.231 DEBUG:teuthology.orchestra.run.smithi131:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-02-19T22:07:49.251 DEBUG:teuthology.orchestra.run.smithi196:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-02-19T22:07:49.328 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-02-19T22:07:49.328 DEBUG:teuthology.orchestra.run.smithi131:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-02-19T22:07:49.330 DEBUG:teuthology.orchestra.run.smithi196:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-02-19T22:07:49.458 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-02-19T22:07:49.469 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-02-19T22:07:49.470 DEBUG:teuthology.orchestra.run.smithi131:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-02-19T22:07:49.497 DEBUG:teuthology.orchestra.run.smithi196:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-02-19T22:07:49.524 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-02-19T22:07:49.535 DEBUG:teuthology.orchestra.run.smithi131:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-02-19T22:07:49.539 DEBUG:teuthology.orchestra.run.smithi196:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-02-19T22:07:49.563 INFO:teuthology.orchestra.run.smithi131.stdout:kernel.core_pattern = core 2024-02-19T22:07:49.590 INFO:teuthology.orchestra.run.smithi196.stdout:kernel.core_pattern = core 2024-02-19T22:07:49.622 DEBUG:teuthology.orchestra.run.smithi131:> test -e /home/ubuntu/cephtest/archive/coredump 2024-02-19T22:07:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-19T22:07:49.651 DEBUG:teuthology.orchestra.run.smithi196:> test -e /home/ubuntu/cephtest/archive/coredump 2024-02-19T22:07:49.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-19T22:07:49.679 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-02-19T22:07:49.690 INFO:teuthology.task.internal:Transferring archived files... 2024-02-19T22:07:49.691 DEBUG:teuthology.misc:Transferring archived files from smithi131:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773/remote/smithi131 2024-02-19T22:07:49.692 DEBUG:teuthology.orchestra.run.smithi131:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-02-19T22:07:49.735 DEBUG:teuthology.misc:Transferring archived files from smithi196:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773/remote/smithi196 2024-02-19T22:07:49.736 DEBUG:teuthology.orchestra.run.smithi196:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-02-19T22:07:49.777 INFO:teuthology.task.internal:Removing archive directory... 2024-02-19T22:07:49.778 DEBUG:teuthology.orchestra.run.smithi131:> rm -rf -- /home/ubuntu/cephtest/archive 2024-02-19T22:07:49.780 DEBUG:teuthology.orchestra.run.smithi196:> rm -rf -- /home/ubuntu/cephtest/archive 2024-02-19T22:07:49.829 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-02-19T22:07:49.841 INFO:teuthology.task.internal:Not uploading archives. 2024-02-19T22:07:49.841 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-02-19T22:07:49.851 INFO:teuthology.task.internal:Tidying up after the test... 2024-02-19T22:07:49.851 DEBUG:teuthology.orchestra.run.smithi131:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-02-19T22:07:49.853 DEBUG:teuthology.orchestra.run.smithi196:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-02-19T22:07:49.868 INFO:teuthology.orchestra.run.smithi131.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 19 22:07 /home/ubuntu/cephtest 2024-02-19T22:07:49.886 INFO:teuthology.orchestra.run.smithi196.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 19 22:07 /home/ubuntu/cephtest 2024-02-19T22:07:49.887 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-02-19T22:07:49.898 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-02-19T22:07:49.920 INFO:teuthology.nuke:Checking targets against current locks 2024-02-19T22:07:49.954 DEBUG:teuthology.nuke:shortname: smithi131 2024-02-19T22:07:49.955 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-19T22:07:50.018 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi131.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-19 21:28:36.237824', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAgG0jlF18ihbqFbZzwpagx6+jBYHXzsEII8carzTu7Vw14Qs9Zo+UhC/Py6tfNnPNg7/sVImU3iV+Oq5Wzpj5I='} 2024-02-19T22:07:50.029 DEBUG:teuthology.nuke:shortname: smithi196 2024-02-19T22:07:50.030 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-19T22:07:50.060 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi196.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-02-19_19:25:49-rados-pacific-release-distro-default-smithi/7566773', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-19 21:28:36.236132', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPFvA3EACYn4fifLnkd2R05kSwjMJ7iZwClMGYRZBS3L2isJojg/yFH9RzeBx+ZqIrNJlHzLANMvIdsslznGB9s='} 2024-02-19T22:07:50.105 INFO:teuthology.orchestra.console.smithi131:Power off 2024-02-19T22:07:50.106 DEBUG:teuthology.orchestra.console.smithi131:pexpect command: ipmitool -H smithi131.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-02-19T22:07:50.152 INFO:teuthology.orchestra.console.smithi196:Power off 2024-02-19T22:07:50.153 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-02-19T22:07:50.183 DEBUG:teuthology.orchestra.console.smithi131:power off output: Chassis Power Control: Down/Off 2024-02-19T22:07:50.183 DEBUG:teuthology.orchestra.console.smithi131:pexpect command: ipmitool -H smithi131.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-19T22:07:50.235 DEBUG:teuthology.orchestra.console.smithi196:power off output: Chassis Power Control: Down/Off 2024-02-19T22:07:50.236 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-19T22:07:50.251 DEBUG:teuthology.orchestra.console.smithi131:check power output: Chassis Power is on 2024-02-19T22:07:50.308 DEBUG:teuthology.orchestra.console.smithi196:check power output: Chassis Power is on 2024-02-19T22:07:54.255 DEBUG:teuthology.orchestra.console.smithi131:pexpect command: ipmitool -H smithi131.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-19T22:07:54.309 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-19T22:07:54.366 DEBUG:teuthology.orchestra.console.smithi131:check power output: Chassis Power is on 2024-02-19T22:07:54.418 DEBUG:teuthology.orchestra.console.smithi196:check power output: Chassis Power is on 2024-02-19T22:07:58.368 DEBUG:teuthology.orchestra.console.smithi131:pexpect command: ipmitool -H smithi131.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-19T22:07:58.419 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-19T22:07:58.477 DEBUG:teuthology.orchestra.console.smithi131:check power output: Chassis Power is on 2024-02-19T22:07:58.528 DEBUG:teuthology.orchestra.console.smithi196:check power output: Chassis Power is on 2024-02-19T22:08:02.482 DEBUG:teuthology.orchestra.console.smithi131:pexpect command: ipmitool -H smithi131.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-19T22:08:02.530 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-19T22:08:02.593 DEBUG:teuthology.orchestra.console.smithi131:check power output: Chassis Power is off 2024-02-19T22:08:02.640 DEBUG:teuthology.orchestra.console.smithi196:check power output: Chassis Power is off 2024-02-19T22:08:02.695 INFO:teuthology.orchestra.console.smithi131:Power off completed 2024-02-19T22:08:02.741 INFO:teuthology.orchestra.console.smithi196:Power off completed 2024-02-19T22:08:02.845 INFO:teuthology.run:Summary data: description: rados/cephadm/mgr-nfs-upgrade/{0-distro/centos_8.stream_container_tools 1-bootstrap/16.2.5 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1845.8891525268555 failure_reason: '"2024-02-19T21:48:05.042519+0000 mon.smithi131 (mon.0) 350 : cluster [WRN] Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON)" in cluster log' owner: scheduled_yuriw@teuthology success: false 2024-02-19T22:08:02.846 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-02-19T22:08:02.934 INFO:teuthology.run:FAIL