2024-02-07T12:06:00.511 INFO:root:teuthology version: 0.0.1.dev256+gd9fdb22 2024-02-07T12:06:00.512 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --owner scheduled_gabrioux@teuthology --archive /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725 -- /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725/orig.config.yaml 2024-02-07T12:06:00.549 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-02-07T12:06:00.777 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725 branch: wip-guits-testing-4-2024-02-05-1854 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7548725' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-testing-4-2024-02-05-1854 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-testing-4-2024-02-05-1854 sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 owner: scheduled_gabrioux@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 1701 sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-testing-4-2024-02-05-1854 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_44458ccedb18241cc2e17f10cdab3d1385399261/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 targets: smithi174.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHGHDzVTMmvRCe98+h4WjUrmQsWynO+p1JNnVlkSvZSdLvFqEUqU9ip9vRdvep7DLjAPaCcdN8d7jdl/0UMK7Qw= smithi184.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjya6K0fUqaU5b91JgaID3NERPMzvfIW0l2bJaoxsbz9RvWVhoAa6Dr2icZzARkaQ30DsvaY/siZxF6JxrsA4A= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: d9fdb2209e15b39d9f061fd85399f352ce0f0894 timestamp: 2024-02-06_08:19:47 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-02-07T12:06:00.777 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_44458ccedb18241cc2e17f10cdab3d1385399261/qa; will attempt to use it 2024-02-07T12:06:00.778 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_44458ccedb18241cc2e17f10cdab3d1385399261/qa/tasks 2024-02-07T12:06:00.778 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-02-07T12:06:00.780 INFO:teuthology.task.internal:Checking packages... 2024-02-07T12:06:00.808 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '44458ccedb18241cc2e17f10cdab3d1385399261' 2024-02-07T12:06:00.808 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-02-07T12:06:00.808 INFO:teuthology.packaging:ref: None 2024-02-07T12:06:00.808 INFO:teuthology.packaging:tag: None 2024-02-07T12:06:00.808 INFO:teuthology.packaging:branch: wip-guits-testing-4-2024-02-05-1854 2024-02-07T12:06:00.808 INFO:teuthology.packaging:sha1: 44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:06:00.808 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-testing-4-2024-02-05-1854 2024-02-07T12:06:01.032 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1252.g44458cce 2024-02-07T12:06:01.033 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-02-07T12:06:01.040 INFO:teuthology.task.internal:no buildpackages task found 2024-02-07T12:06:01.040 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-02-07T12:06:01.099 INFO:teuthology.task.internal:Saving configuration 2024-02-07T12:06:01.113 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-02-07T12:06:01.119 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-07T12:06:01.146 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi174.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-07 12:00:52.249965', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHGHDzVTMmvRCe98+h4WjUrmQsWynO+p1JNnVlkSvZSdLvFqEUqU9ip9vRdvep7DLjAPaCcdN8d7jdl/0UMK7Qw='} 2024-02-07T12:06:01.174 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi184.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-07 12:00:52.251928', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjya6K0fUqaU5b91JgaID3NERPMzvfIW0l2bJaoxsbz9RvWVhoAa6Dr2icZzARkaQ30DsvaY/siZxF6JxrsA4A='} 2024-02-07T12:06:01.174 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-02-07T12:06:01.181 INFO:teuthology.task.internal:roles: ubuntu@smithi174.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-02-07T12:06:01.181 INFO:teuthology.task.internal:roles: ubuntu@smithi184.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-02-07T12:06:01.181 INFO:teuthology.run_tasks:Running task console_log... 2024-02-07T12:06:01.287 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f69f253df70>, signals=[15]) 2024-02-07T12:06:01.287 INFO:teuthology.run_tasks:Running task internal.connect... 2024-02-07T12:06:01.294 INFO:teuthology.task.internal:Opening connections... 2024-02-07T12:06:01.295 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:06:01.298 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:06:01.392 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi184.front.sepia.ceph.com 2024-02-07T12:06:01.393 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi184.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:06:01.485 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-02-07T12:06:01.492 DEBUG:teuthology.orchestra.run.smithi174:> uname -m 2024-02-07T12:06:01.510 INFO:teuthology.orchestra.run.smithi174.stdout:x86_64 2024-02-07T12:06:01.511 DEBUG:teuthology.orchestra.run.smithi174:> cat /etc/os-release 2024-02-07T12:06:01.572 INFO:teuthology.orchestra.run.smithi174.stdout:NAME="CentOS Stream" 2024-02-07T12:06:01.572 INFO:teuthology.orchestra.run.smithi174.stdout:VERSION="9" 2024-02-07T12:06:01.572 INFO:teuthology.orchestra.run.smithi174.stdout:ID="centos" 2024-02-07T12:06:01.573 INFO:teuthology.orchestra.run.smithi174.stdout:ID_LIKE="rhel fedora" 2024-02-07T12:06:01.573 INFO:teuthology.orchestra.run.smithi174.stdout:VERSION_ID="9" 2024-02-07T12:06:01.573 INFO:teuthology.orchestra.run.smithi174.stdout:PLATFORM_ID="platform:el9" 2024-02-07T12:06:01.573 INFO:teuthology.orchestra.run.smithi174.stdout:PRETTY_NAME="CentOS Stream 9" 2024-02-07T12:06:01.573 INFO:teuthology.orchestra.run.smithi174.stdout:ANSI_COLOR="0;31" 2024-02-07T12:06:01.573 INFO:teuthology.orchestra.run.smithi174.stdout:LOGO="fedora-logo-icon" 2024-02-07T12:06:01.573 INFO:teuthology.orchestra.run.smithi174.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-02-07T12:06:01.574 INFO:teuthology.orchestra.run.smithi174.stdout:HOME_URL="https://centos.org/" 2024-02-07T12:06:01.574 INFO:teuthology.orchestra.run.smithi174.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-02-07T12:06:01.574 INFO:teuthology.orchestra.run.smithi174.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-02-07T12:06:01.574 INFO:teuthology.orchestra.run.smithi174.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-02-07T12:06:01.574 INFO:teuthology.lock.ops:Updating smithi174.front.sepia.ceph.com on lock server 2024-02-07T12:06:01.596 DEBUG:teuthology.orchestra.run.smithi184:> uname -m 2024-02-07T12:06:01.613 INFO:teuthology.orchestra.run.smithi184.stdout:x86_64 2024-02-07T12:06:01.614 DEBUG:teuthology.orchestra.run.smithi184:> cat /etc/os-release 2024-02-07T12:06:01.670 INFO:teuthology.orchestra.run.smithi184.stdout:NAME="CentOS Stream" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:VERSION="9" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:ID="centos" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:ID_LIKE="rhel fedora" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:VERSION_ID="9" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:PLATFORM_ID="platform:el9" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:PRETTY_NAME="CentOS Stream 9" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:ANSI_COLOR="0;31" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:LOGO="fedora-logo-icon" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:HOME_URL="https://centos.org/" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-02-07T12:06:01.671 INFO:teuthology.orchestra.run.smithi184.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-02-07T12:06:01.672 INFO:teuthology.lock.ops:Updating smithi184.front.sepia.ceph.com on lock server 2024-02-07T12:06:01.696 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-02-07T12:06:01.705 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-02-07T12:06:01.711 INFO:teuthology.task.internal:Checking for old test directory... 2024-02-07T12:06:01.712 DEBUG:teuthology.orchestra.run.smithi174:> test '!' -e /home/ubuntu/cephtest 2024-02-07T12:06:01.714 DEBUG:teuthology.orchestra.run.smithi184:> test '!' -e /home/ubuntu/cephtest 2024-02-07T12:06:01.730 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-02-07T12:06:01.736 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-02-07T12:06:01.736 DEBUG:teuthology.orchestra.run.smithi174:> test -z $(ls -A /var/lib/ceph) 2024-02-07T12:06:01.774 DEBUG:teuthology.orchestra.run.smithi184:> test -z $(ls -A /var/lib/ceph) 2024-02-07T12:06:01.808 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-02-07T12:06:01.863 INFO:teuthology.run_tasks:Running task kernel... 2024-02-07T12:06:01.881 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-02-07T12:06:01.881 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-02-07T12:06:01.881 DEBUG:teuthology.orchestra.run.smithi174:> test -f /run/.containerenv -o -f /.dockerenv 2024-02-07T12:06:01.882 DEBUG:teuthology.orchestra.run.smithi184:> test -f /run/.containerenv -o -f /.dockerenv 2024-02-07T12:06:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:06:01.897 DEBUG:teuthology.orchestra.run.smithi174:> uname -r 2024-02-07T12:06:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:06:01.898 DEBUG:teuthology.orchestra.run.smithi184:> uname -r 2024-02-07T12:06:01.954 INFO:teuthology.orchestra.run.smithi184.stdout:5.14.0-361.el9.x86_64 2024-02-07T12:06:01.955 INFO:teuthology.task.kernel:Running kernel on smithi184: 5.14.0-361.el9.x86_64 2024-02-07T12:06:01.955 DEBUG:teuthology.orchestra.run.smithi184:> sudo yum install -y kernel 2024-02-07T12:06:01.957 INFO:teuthology.orchestra.run.smithi174.stdout:5.14.0-361.el9.x86_64 2024-02-07T12:06:01.957 INFO:teuthology.task.kernel:Running kernel on smithi174: 5.14.0-361.el9.x86_64 2024-02-07T12:06:01.957 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum install -y kernel 2024-02-07T12:06:04.041 INFO:teuthology.orchestra.run.smithi184.stdout:CentOS-9 - AppStream 16 MB/s | 6.7 MB 00:00 2024-02-07T12:06:04.152 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS-9 - AppStream 13 MB/s | 6.7 MB 00:00 2024-02-07T12:06:05.416 INFO:teuthology.orchestra.run.smithi184.stdout:CentOS-9 - BaseOS 7.4 MB/s | 2.0 MB 00:00 2024-02-07T12:06:05.531 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS-9 - BaseOS 7.9 MB/s | 2.0 MB 00:00 2024-02-07T12:06:06.665 INFO:teuthology.orchestra.run.smithi184.stdout:CentOS Stream 9 - BaseOS 8.8 MB/s | 8.0 MB 00:00 2024-02-07T12:06:06.747 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - BaseOS 9.0 MB/s | 8.0 MB 00:00 2024-02-07T12:06:08.575 INFO:teuthology.orchestra.run.smithi184.stdout:CentOS Stream 9 - AppStream 22 MB/s | 19 MB 00:00 2024-02-07T12:06:08.725 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - AppStream 21 MB/s | 19 MB 00:00 2024-02-07T12:06:12.751 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - CRB 11 MB/s | 5.9 MB 00:00 2024-02-07T12:06:13.444 INFO:teuthology.orchestra.run.smithi184.stdout:CentOS Stream 9 - CRB 4.3 MB/s | 5.9 MB 00:01 2024-02-07T12:06:14.291 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - Extras packages 34 kB/s | 15 kB 00:00 2024-02-07T12:06:14.959 INFO:teuthology.orchestra.run.smithi184.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 15 kB 00:00 2024-02-07T12:06:15.904 INFO:teuthology.orchestra.run.smithi184.stdout:Extra Packages for Enterprise Linux 25 MB/s | 20 MB 00:00 2024-02-07T12:06:16.245 INFO:teuthology.orchestra.run.smithi174.stdout:Extra Packages for Enterprise Linux 11 MB/s | 20 MB 00:01 2024-02-07T12:06:20.802 INFO:teuthology.orchestra.run.smithi184.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-02-07T12:06:21.050 INFO:teuthology.orchestra.run.smithi174.stdout:lab-extras 28 kB/s | 1.7 kB 00:00 2024-02-07T12:06:23.051 INFO:teuthology.orchestra.run.smithi184.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-02-07T12:06:23.051 INFO:teuthology.orchestra.run.smithi184.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-02-07T12:06:23.139 INFO:teuthology.orchestra.run.smithi184.stdout:Dependencies resolved. 2024-02-07T12:06:23.149 INFO:teuthology.orchestra.run.smithi184.stdout:================================================================================ 2024-02-07T12:06:23.149 INFO:teuthology.orchestra.run.smithi184.stdout: Package Arch Version Repository Size 2024-02-07T12:06:23.149 INFO:teuthology.orchestra.run.smithi184.stdout:================================================================================ 2024-02-07T12:06:23.149 INFO:teuthology.orchestra.run.smithi184.stdout:Installing: 2024-02-07T12:06:23.149 INFO:teuthology.orchestra.run.smithi184.stdout: kernel x86_64 5.14.0-418.el9 CentOS-BaseOS 6.0 M 2024-02-07T12:06:23.149 INFO:teuthology.orchestra.run.smithi184.stdout:Installing dependencies: 2024-02-07T12:06:23.150 INFO:teuthology.orchestra.run.smithi184.stdout: kernel-core x86_64 5.14.0-418.el9 CentOS-BaseOS 21 M 2024-02-07T12:06:23.150 INFO:teuthology.orchestra.run.smithi184.stdout: kernel-modules x86_64 5.14.0-418.el9 CentOS-BaseOS 39 M 2024-02-07T12:06:23.150 INFO:teuthology.orchestra.run.smithi184.stdout: kernel-modules-core x86_64 5.14.0-418.el9 CentOS-BaseOS 33 M 2024-02-07T12:06:23.150 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:06:23.150 INFO:teuthology.orchestra.run.smithi184.stdout:Transaction Summary 2024-02-07T12:06:23.150 INFO:teuthology.orchestra.run.smithi184.stdout:================================================================================ 2024-02-07T12:06:23.150 INFO:teuthology.orchestra.run.smithi184.stdout:Install 4 Packages 2024-02-07T12:06:23.150 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:06:23.151 INFO:teuthology.orchestra.run.smithi184.stdout:Total download size: 99 M 2024-02-07T12:06:23.151 INFO:teuthology.orchestra.run.smithi184.stdout:Installed size: 122 M 2024-02-07T12:06:23.151 INFO:teuthology.orchestra.run.smithi184.stdout:Downloading Packages: 2024-02-07T12:06:23.265 INFO:teuthology.orchestra.run.smithi174.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-02-07T12:06:23.268 INFO:teuthology.orchestra.run.smithi174.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-02-07T12:06:23.367 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2024-02-07T12:06:23.372 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-02-07T12:06:23.372 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2024-02-07T12:06:23.372 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout: kernel x86_64 5.14.0-418.el9 CentOS-BaseOS 6.0 M 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-core x86_64 5.14.0-418.el9 CentOS-BaseOS 21 M 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules x86_64 5.14.0-418.el9 CentOS-BaseOS 39 M 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-core x86_64 5.14.0-418.el9 CentOS-BaseOS 33 M 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout:Install 4 Packages 2024-02-07T12:06:23.373 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:06:23.374 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 99 M 2024-02-07T12:06:23.374 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 122 M 2024-02-07T12:06:23.374 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2024-02-07T12:06:23.545 INFO:teuthology.orchestra.run.smithi184.stdout:(1/4): kernel-5.14.0-418.el9.x86_64.rpm 15 MB/s | 6.0 MB 00:00 2024-02-07T12:06:24.010 INFO:teuthology.orchestra.run.smithi174.stdout:(1/4): kernel-5.14.0-418.el9.x86_64.rpm 9.4 MB/s | 6.0 MB 00:00 2024-02-07T12:06:24.079 INFO:teuthology.orchestra.run.smithi184.stdout:(2/4): kernel-core-5.14.0-418.el9.x86_64.rpm 22 MB/s | 21 MB 00:00 2024-02-07T12:06:24.802 INFO:teuthology.orchestra.run.smithi174.stdout:(2/4): kernel-core-5.14.0-418.el9.x86_64.rpm 15 MB/s | 21 MB 00:01 2024-02-07T12:06:25.195 INFO:teuthology.orchestra.run.smithi184.stdout:(3/4): kernel-modules-5.14.0-418.el9.x86_64.rpm 19 MB/s | 39 MB 00:02 2024-02-07T12:06:25.310 INFO:teuthology.orchestra.run.smithi174.stdout:(3/4): kernel-modules-core-5.14.0-418.el9.x86_6 26 MB/s | 33 MB 00:01 2024-02-07T12:06:25.554 INFO:teuthology.orchestra.run.smithi184.stdout:(4/4): kernel-modules-core-5.14.0-418.el9.x86_6 17 MB/s | 33 MB 00:02 2024-02-07T12:06:25.557 INFO:teuthology.orchestra.run.smithi184.stdout:-------------------------------------------------------------------------------- 2024-02-07T12:06:25.557 INFO:teuthology.orchestra.run.smithi184.stdout:Total 41 MB/s | 99 MB 00:02 2024-02-07T12:06:25.557 INFO:teuthology.orchestra.run.smithi184.stdout:Running transaction check 2024-02-07T12:06:25.902 INFO:teuthology.orchestra.run.smithi184.stdout:Transaction check succeeded. 2024-02-07T12:06:25.902 INFO:teuthology.orchestra.run.smithi184.stdout:Running transaction test 2024-02-07T12:06:25.960 INFO:teuthology.orchestra.run.smithi174.stdout:(4/4): kernel-modules-5.14.0-418.el9.x86_64.rpm 15 MB/s | 39 MB 00:02 2024-02-07T12:06:25.964 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2024-02-07T12:06:25.964 INFO:teuthology.orchestra.run.smithi174.stdout:Total 38 MB/s | 99 MB 00:02 2024-02-07T12:06:25.964 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2024-02-07T12:06:26.323 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2024-02-07T12:06:26.323 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2024-02-07T12:06:26.902 INFO:teuthology.orchestra.run.smithi184.stdout:Transaction test succeeded. 2024-02-07T12:06:26.902 INFO:teuthology.orchestra.run.smithi184.stdout:Running transaction 2024-02-07T12:06:27.334 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2024-02-07T12:06:27.335 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2024-02-07T12:06:29.297 INFO:teuthology.orchestra.run.smithi184.stdout: Preparing : 1/1 2024-02-07T12:06:29.740 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2024-02-07T12:06:29.957 INFO:teuthology.orchestra.run.smithi184.stdout: Installing : kernel-modules-core-5.14.0-418.el9.x86_64 1/4 2024-02-07T12:06:30.096 INFO:teuthology.orchestra.run.smithi184.stdout: Installing : kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:06:30.404 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-modules-core-5.14.0-418.el9.x86_64 1/4 2024-02-07T12:06:30.535 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:06:32.846 INFO:teuthology.orchestra.run.smithi184.stdout: Running scriptlet: kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:06:33.034 INFO:teuthology.orchestra.run.smithi184.stdout: Installing : kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:06:33.291 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:06:33.479 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:06:37.751 INFO:teuthology.orchestra.run.smithi184.stdout: Running scriptlet: kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:06:38.047 INFO:teuthology.orchestra.run.smithi184.stdout: Installing : kernel-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:06:38.251 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:06:38.357 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:06:42.790 INFO:teuthology.orchestra.run.smithi184.stdout: Running scriptlet: kernel-modules-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:06:43.151 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:12.076 INFO:teuthology.orchestra.run.smithi184.stdout: Running scriptlet: kernel-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:12.076 INFO:teuthology.orchestra.run.smithi184.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-02-07T12:07:12.076 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:07:12.147 INFO:teuthology.orchestra.run.smithi184.stdout: Running scriptlet: kernel-modules-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:12.217 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:12.217 INFO:teuthology.orchestra.run.smithi174.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-02-07T12:07:12.217 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:07:12.287 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:12.633 INFO:teuthology.orchestra.run.smithi184.stdout: Running scriptlet: kernel-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:12.633 INFO:teuthology.orchestra.run.smithi184.stdout: Verifying : kernel-5.14.0-418.el9.x86_64 1/4 2024-02-07T12:07:12.634 INFO:teuthology.orchestra.run.smithi184.stdout: Verifying : kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:07:12.634 INFO:teuthology.orchestra.run.smithi184.stdout: Verifying : kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:07:12.681 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:12.681 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-5.14.0-418.el9.x86_64 1/4 2024-02-07T12:07:12.681 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-core-5.14.0-418.el9.x86_64 2/4 2024-02-07T12:07:12.681 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-modules-5.14.0-418.el9.x86_64 3/4 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout: Verifying : kernel-modules-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout:Installed: 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout: kernel-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout: kernel-core-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout: kernel-modules-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout: kernel-modules-core-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:07:13.058 INFO:teuthology.orchestra.run.smithi184.stdout:Complete! 2024-02-07T12:07:13.118 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-modules-core-5.14.0-418.el9.x86_64 4/4 2024-02-07T12:07:13.118 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:07:13.118 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2024-02-07T12:07:13.118 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.118 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-core-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.118 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.118 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-core-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.119 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:07:13.119 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2024-02-07T12:07:13.538 DEBUG:teuthology.orchestra.run.smithi184:> rpm -q kernel | sort -rV | head -n 1 2024-02-07T12:07:13.586 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q kernel | sort -rV | head -n 1 2024-02-07T12:07:13.669 INFO:teuthology.orchestra.run.smithi184.stdout:kernel-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.670 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-418.el9.x86_64 2024-02-07T12:07:13.670 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-418.el9.x86_64 2024-02-07T12:07:13.670 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-02-07T12:07:13.670 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-02-07T12:07:13.670 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-02-07T12:07:13.670 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi184.front.sepia.ceph.com, path=None, version=distro) 2024-02-07T12:07:13.670 DEBUG:teuthology.orchestra.run.smithi184:> rpm -q kernel | sort -rV | head -n 1 2024-02-07T12:07:13.719 INFO:teuthology.orchestra.run.smithi174.stdout:kernel-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.719 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-418.el9.x86_64 2024-02-07T12:07:13.719 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-418.el9.x86_64 2024-02-07T12:07:13.719 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-02-07T12:07:13.719 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-02-07T12:07:13.719 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-02-07T12:07:13.720 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi174.front.sepia.ceph.com, path=None, version=distro) 2024-02-07T12:07:13.720 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q kernel | sort -rV | head -n 1 2024-02-07T12:07:13.779 INFO:teuthology.orchestra.run.smithi184.stdout:kernel-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.779 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-418.el9.x86_64 2024-02-07T12:07:13.780 DEBUG:teuthology.orchestra.run.smithi184:> sudo rpm -qi grub2-tools 2024-02-07T12:07:13.818 INFO:teuthology.orchestra.run.smithi184.stdout:Name : grub2-tools 2024-02-07T12:07:13.818 INFO:teuthology.orchestra.run.smithi184.stdout:Epoch : 1 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Version : 2.06 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Release : 68.el9 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Architecture: x86_64 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Group : Unspecified 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Size : 8264254 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:License : GPLv3+ 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Packager : builder@centos.org 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Vendor : CentOS 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:URL : http://www.gnu.org/software/grub/ 2024-02-07T12:07:13.819 INFO:teuthology.orchestra.run.smithi184.stdout:Summary : Support tools for GRUB. 2024-02-07T12:07:13.820 INFO:teuthology.orchestra.run.smithi184.stdout:Description : 2024-02-07T12:07:13.820 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:07:13.820 INFO:teuthology.orchestra.run.smithi184.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-02-07T12:07:13.820 INFO:teuthology.orchestra.run.smithi184.stdout:customizable bootloader with modular architecture. It supports a rich 2024-02-07T12:07:13.820 INFO:teuthology.orchestra.run.smithi184.stdout:variety of kernel formats, file systems, computer architectures and 2024-02-07T12:07:13.820 INFO:teuthology.orchestra.run.smithi184.stdout:hardware devices. 2024-02-07T12:07:13.820 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:07:13.820 INFO:teuthology.orchestra.run.smithi184.stdout:This subpackage provides tools for support of all platforms. 2024-02-07T12:07:13.821 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-02-07T12:07:13.821 INFO:teuthology.task.kernel:Updating grub on smithi184 to boot 5.14.0-418.el9.x86_64 2024-02-07T12:07:13.821 DEBUG:teuthology.orchestra.run.smithi184:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-02-07T12:07:13.827 INFO:teuthology.orchestra.run.smithi174.stdout:kernel-5.14.0-418.el9.x86_64 2024-02-07T12:07:13.828 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-418.el9.x86_64 2024-02-07T12:07:13.828 DEBUG:teuthology.orchestra.run.smithi174:> sudo rpm -qi grub2-tools 2024-02-07T12:07:13.864 INFO:teuthology.orchestra.run.smithi174.stdout:Name : grub2-tools 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Epoch : 1 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Version : 2.06 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Release : 68.el9 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Architecture: x86_64 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Group : Unspecified 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Size : 8264254 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:License : GPLv3+ 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-02-07T12:07:13.865 INFO:teuthology.orchestra.run.smithi174.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:Packager : builder@centos.org 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:Vendor : CentOS 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:URL : http://www.gnu.org/software/grub/ 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:Summary : Support tools for GRUB. 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:Description : 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:customizable bootloader with modular architecture. It supports a rich 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:variety of kernel formats, file systems, computer architectures and 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:hardware devices. 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:07:13.866 INFO:teuthology.orchestra.run.smithi174.stdout:This subpackage provides tools for support of all platforms. 2024-02-07T12:07:13.867 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-02-07T12:07:13.868 INFO:teuthology.task.kernel:Updating grub on smithi174 to boot 5.14.0-418.el9.x86_64 2024-02-07T12:07:13.868 DEBUG:teuthology.orchestra.run.smithi174:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-02-07T12:07:14.383 INFO:teuthology.orchestra.run.smithi184.stderr:Generating grub configuration file ... 2024-02-07T12:07:14.486 INFO:teuthology.orchestra.run.smithi174.stderr:Generating grub configuration file ... 2024-02-07T12:07:15.819 INFO:teuthology.orchestra.run.smithi184.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-02-07T12:07:15.860 INFO:teuthology.orchestra.run.smithi184.stderr:done 2024-02-07T12:07:15.863 DEBUG:teuthology.orchestra.run.smithi184:> mktemp 2024-02-07T12:07:15.881 INFO:teuthology.orchestra.run.smithi184.stdout:/tmp/tmp.JNnhWU2iHM 2024-02-07T12:07:15.881 DEBUG:teuthology.orchestra.run.smithi184:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.JNnhWU2iHM 2024-02-07T12:07:15.891 INFO:teuthology.orchestra.run.smithi174.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-02-07T12:07:15.918 INFO:teuthology.orchestra.run.smithi174.stderr:done 2024-02-07T12:07:15.920 DEBUG:teuthology.orchestra.run.smithi174:> mktemp 2024-02-07T12:07:15.935 INFO:teuthology.orchestra.run.smithi174.stdout:/tmp/tmp.WylxIYzXcJ 2024-02-07T12:07:15.935 DEBUG:teuthology.orchestra.run.smithi174:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.WylxIYzXcJ 2024-02-07T12:07:15.961 DEBUG:teuthology.orchestra.run.smithi184:> sudo chmod 0666 /tmp/tmp.JNnhWU2iHM 2024-02-07T12:07:16.004 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 0666 /tmp/tmp.WylxIYzXcJ 2024-02-07T12:07:16.105 DEBUG:teuthology.orchestra.remote:smithi184:/tmp/tmp.JNnhWU2iHM is 6KB 2024-02-07T12:07:16.135 DEBUG:teuthology.orchestra.remote:smithi174:/tmp/tmp.WylxIYzXcJ is 6KB 2024-02-07T12:07:16.153 DEBUG:teuthology.orchestra.run.smithi184:> rm -fr /tmp/tmp.JNnhWU2iHM 2024-02-07T12:07:16.166 DEBUG:teuthology.orchestra.run.smithi184:> sudo /bin/ls /boot/loader/entries || true 2024-02-07T12:07:16.183 DEBUG:teuthology.orchestra.run.smithi174:> rm -fr /tmp/tmp.WylxIYzXcJ 2024-02-07T12:07:16.197 DEBUG:teuthology.orchestra.run.smithi174:> sudo /bin/ls /boot/loader/entries || true 2024-02-07T12:07:16.230 INFO:teuthology.orchestra.run.smithi184.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-02-07T12:07:16.230 INFO:teuthology.orchestra.run.smithi184.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-02-07T12:07:16.230 INFO:teuthology.orchestra.run.smithi184.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-02-07T12:07:16.230 INFO:teuthology.orchestra.run.smithi184.stdout:15949e3c88704300882482eddfd7baae-5.14.0-418.el9.x86_64.conf 2024-02-07T12:07:16.232 DEBUG:teuthology.orchestra.run.smithi184:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-418.el9.x86_64 2024-02-07T12:07:16.262 INFO:teuthology.orchestra.run.smithi174.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-02-07T12:07:16.262 INFO:teuthology.orchestra.run.smithi174.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-02-07T12:07:16.262 INFO:teuthology.orchestra.run.smithi174.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-02-07T12:07:16.262 INFO:teuthology.orchestra.run.smithi174.stdout:15949e3c88704300882482eddfd7baae-5.14.0-418.el9.x86_64.conf 2024-02-07T12:07:16.263 DEBUG:teuthology.orchestra.run.smithi174:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-418.el9.x86_64 2024-02-07T12:07:16.381 DEBUG:teuthology.orchestra.run.smithi184:> sudo shutdown -r now 2024-02-07T12:07:16.416 DEBUG:teuthology.orchestra.run.smithi174:> sudo shutdown -r now 2024-02-07T12:07:46.410 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-02-07T12:07:46.411 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi184.front.sepia.ceph.com' 2024-02-07T12:07:46.412 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi184.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:07:46.419 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-02-07T12:07:46.419 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-02-07T12:07:46.419 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:08:04.928 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.184 2024-02-07T12:08:04.929 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.174 2024-02-07T12:08:13.936 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi184.front.sepia.ceph.com' 2024-02-07T12:08:13.937 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi184.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:08:13.939 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-02-07T12:08:13.939 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:08:17.024 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.174 2024-02-07T12:08:29.036 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-02-07T12:08:29.037 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:08:47.420 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.174 2024-02-07T12:09:02.436 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-02-07T12:09:02.437 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:09:02.857 DEBUG:teuthology.orchestra.run.smithi174:> true 2024-02-07T12:09:03.254 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-02-07T12:09:03.254 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-418.el9.x86_64"... 2024-02-07T12:09:03.255 DEBUG:teuthology.orchestra.run.smithi174:> uname -r 2024-02-07T12:09:03.268 INFO:teuthology.orchestra.run.smithi174.stdout:5.14.0-418.el9.x86_64 2024-02-07T12:09:03.269 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-418.el9.x86_64 vs 5.14.0-418.el9.x86_64 2024-02-07T12:09:03.269 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-02-07T12:09:03.269 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-02-07T12:09:04.269 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-02-07T12:09:04.270 DEBUG:teuthology.orchestra.run.smithi174:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-02-07T12:09:04.337 INFO:teuthology.orchestra.run.smithi174.stdout:ttyS1 2024-02-07T12:09:04.379 DEBUG:teuthology.parallel:result is None 2024-02-07T12:09:13.939 DEBUG:teuthology.orchestra.remote:timed out 2024-02-07T12:09:25.940 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi184.front.sepia.ceph.com' 2024-02-07T12:09:25.942 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi184.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:09:26.207 DEBUG:teuthology.orchestra.run.smithi184:> true 2024-02-07T12:09:26.568 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi184.front.sepia.ceph.com' 2024-02-07T12:09:26.568 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-418.el9.x86_64"... 2024-02-07T12:09:26.569 DEBUG:teuthology.orchestra.run.smithi184:> uname -r 2024-02-07T12:09:26.585 INFO:teuthology.orchestra.run.smithi184.stdout:5.14.0-418.el9.x86_64 2024-02-07T12:09:26.586 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-418.el9.x86_64 vs 5.14.0-418.el9.x86_64 2024-02-07T12:09:26.586 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-02-07T12:09:26.586 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-02-07T12:09:27.587 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-02-07T12:09:27.587 DEBUG:teuthology.orchestra.run.smithi184:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-02-07T12:09:27.667 INFO:teuthology.orchestra.run.smithi184.stdout:ttyS1 2024-02-07T12:09:27.707 DEBUG:teuthology.parallel:result is None 2024-02-07T12:09:27.707 INFO:teuthology.run_tasks:Running task internal.base... 2024-02-07T12:09:27.715 INFO:teuthology.task.internal:Creating test directory... 2024-02-07T12:09:27.715 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-02-07T12:09:27.718 DEBUG:teuthology.orchestra.run.smithi184:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-02-07T12:09:27.735 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-02-07T12:09:27.742 INFO:teuthology.run_tasks:Running task internal.archive... 2024-02-07T12:09:27.751 INFO:teuthology.task.internal:Creating archive directory... 2024-02-07T12:09:27.751 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-02-07T12:09:27.776 DEBUG:teuthology.orchestra.run.smithi184:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-02-07T12:09:27.807 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-02-07T12:09:27.814 INFO:teuthology.task.internal:Enabling coredump saving... 2024-02-07T12:09:27.814 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-02-07T12:09:27.845 DEBUG:teuthology.orchestra.run.smithi184:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-02-07T12:09:27.881 INFO:teuthology.orchestra.run.smithi184.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-07T12:09:27.883 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-07T12:09:27.896 INFO:teuthology.orchestra.run.smithi184.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-07T12:09:27.897 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-02-07T12:09:27.898 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-02-07T12:09:27.906 INFO:teuthology.task.internal:Configuring sudo... 2024-02-07T12:09:27.906 DEBUG:teuthology.orchestra.run.smithi174:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-02-07T12:09:27.941 DEBUG:teuthology.orchestra.run.smithi184:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-02-07T12:09:27.970 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-02-07T12:09:27.980 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-02-07T12:09:27.980 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-02-07T12:09:28.012 DEBUG:teuthology.orchestra.run.smithi184:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-02-07T12:09:28.028 DEBUG:teuthology.orchestra.run.smithi174:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-07T12:09:28.110 DEBUG:teuthology.orchestra.run.smithi174:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-07T12:09:28.160 DEBUG:teuthology.orchestra.run.smithi174:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-07T12:09:28.243 DEBUG:teuthology.orchestra.run.smithi174:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-07T12:09:28.288 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:09:28.289 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-02-07T12:09:28.358 DEBUG:teuthology.orchestra.run.smithi184:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-07T12:09:28.403 DEBUG:teuthology.orchestra.run.smithi184:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-02-07T12:09:28.460 DEBUG:teuthology.orchestra.run.smithi184:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-07T12:09:28.545 DEBUG:teuthology.orchestra.run.smithi184:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-02-07T12:09:28.590 DEBUG:teuthology.orchestra.run.smithi184:> set -ex 2024-02-07T12:09:28.590 DEBUG:teuthology.orchestra.run.smithi184:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-02-07T12:09:28.659 DEBUG:teuthology.orchestra.run.smithi174:> sudo service rsyslog restart 2024-02-07T12:09:28.662 DEBUG:teuthology.orchestra.run.smithi184:> sudo service rsyslog restart 2024-02-07T12:09:28.708 INFO:teuthology.orchestra.run.smithi174.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-07T12:09:28.741 INFO:teuthology.orchestra.run.smithi184.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-07T12:09:29.236 INFO:teuthology.run_tasks:Running task internal.timer... 2024-02-07T12:09:29.244 INFO:teuthology.task.internal:Starting timer... 2024-02-07T12:09:29.244 INFO:teuthology.run_tasks:Running task pcp... 2024-02-07T12:09:29.256 INFO:teuthology.run_tasks:Running task selinux... 2024-02-07T12:09:29.265 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-02-07T12:09:29.266 DEBUG:teuthology.orchestra.run.smithi174:> sudo service auditd rotate 2024-02-07T12:09:29.349 INFO:teuthology.orchestra.run.smithi174.stdout:Rotating logs: 2024-02-07T12:09:29.351 DEBUG:teuthology.orchestra.run.smithi184:> sudo service auditd rotate 2024-02-07T12:09:29.400 INFO:teuthology.orchestra.run.smithi184.stdout:Rotating logs: 2024-02-07T12:09:29.402 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-02-07T12:09:29.402 DEBUG:teuthology.orchestra.run.smithi174:> /usr/sbin/getenforce 2024-02-07T12:09:29.427 INFO:teuthology.orchestra.run.smithi174.stdout:Permissive 2024-02-07T12:09:29.427 DEBUG:teuthology.orchestra.run.smithi184:> /usr/sbin/getenforce 2024-02-07T12:09:29.447 INFO:teuthology.orchestra.run.smithi184.stdout:Permissive 2024-02-07T12:09:29.447 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi174.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi184.front.sepia.ceph.com': 'permissive'} 2024-02-07T12:09:29.447 DEBUG:teuthology.orchestra.run.smithi174:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-07T12:09:29.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:09:29.498 DEBUG:teuthology.orchestra.run.smithi184:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-07T12:09:29.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:09:29.526 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-02-07T12:09:29.526 DEBUG:teuthology.orchestra.run.smithi174:> sudo /usr/sbin/setenforce permissive 2024-02-07T12:09:29.571 DEBUG:teuthology.orchestra.run.smithi184:> sudo /usr/sbin/setenforce permissive 2024-02-07T12:09:29.615 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-02-07T12:09:29.625 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-02-07T12:09:29.631 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-02-07T12:09:29.755 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-02-07T12:09:29.772 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-02-07T12:09:29.773 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi174.front.sepia.ceph.com,smithi184.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-02-07T12:13:38.992 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi174.front.sepia.ceph.com'), Remote(name='ubuntu@smithi184.front.sepia.ceph.com')] 2024-02-07T12:13:38.993 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-02-07T12:13:38.994 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:13:39.091 DEBUG:teuthology.orchestra.run.smithi174:> true 2024-02-07T12:13:39.168 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-02-07T12:13:39.168 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi184.front.sepia.ceph.com' 2024-02-07T12:13:39.170 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi184.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-02-07T12:13:39.263 DEBUG:teuthology.orchestra.run.smithi184:> true 2024-02-07T12:13:39.339 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi184.front.sepia.ceph.com' 2024-02-07T12:13:39.339 INFO:teuthology.run_tasks:Running task clock... 2024-02-07T12:13:39.351 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-02-07T12:13:39.351 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-02-07T12:13:39.351 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-07T12:13:39.354 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-02-07T12:13:39.354 DEBUG:teuthology.orchestra.run.smithi184:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-07T12:13:39.382 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-02-07T12:13:39.398 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-02-07T12:13:39.411 INFO:teuthology.orchestra.run.smithi184.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-02-07T12:13:39.426 INFO:teuthology.orchestra.run.smithi174.stderr:sudo: ntpd: command not found 2024-02-07T12:13:39.428 INFO:teuthology.orchestra.run.smithi184.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-02-07T12:13:39.440 INFO:teuthology.orchestra.run.smithi174.stdout:506 Cannot talk to daemon 2024-02-07T12:13:39.457 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-02-07T12:13:39.458 INFO:teuthology.orchestra.run.smithi184.stderr:sudo: ntpd: command not found 2024-02-07T12:13:39.472 INFO:teuthology.orchestra.run.smithi184.stdout:506 Cannot talk to daemon 2024-02-07T12:13:39.473 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-02-07T12:13:39.489 INFO:teuthology.orchestra.run.smithi184.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-02-07T12:13:39.505 INFO:teuthology.orchestra.run.smithi184.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-02-07T12:13:39.525 INFO:teuthology.orchestra.run.smithi174.stderr:bash: line 1: ntpq: command not found 2024-02-07T12:13:39.528 INFO:teuthology.orchestra.run.smithi174.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-07T12:13:39.528 INFO:teuthology.orchestra.run.smithi174.stdout:=============================================================================== 2024-02-07T12:13:39.528 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:13:39.528 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:13:39.528 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:13:39.528 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:13:39.558 INFO:teuthology.orchestra.run.smithi184.stderr:bash: line 1: ntpq: command not found 2024-02-07T12:13:39.561 INFO:teuthology.orchestra.run.smithi184.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-07T12:13:39.561 INFO:teuthology.orchestra.run.smithi184.stdout:=============================================================================== 2024-02-07T12:13:39.561 INFO:teuthology.orchestra.run.smithi184.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:13:39.561 INFO:teuthology.orchestra.run.smithi184.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:13:39.561 INFO:teuthology.orchestra.run.smithi184.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:13:39.561 INFO:teuthology.orchestra.run.smithi184.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:13:39.562 INFO:teuthology.run_tasks:Running task cephadm... 2024-02-07T12:13:39.665 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '44458ccedb18241cc2e17f10cdab3d1385399261'} 2024-02-07T12:13:39.665 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-02-07T12:13:39.665 INFO:tasks.cephadm:Cluster fsid is 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:13:39.665 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-02-07T12:13:39.665 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-02-07T12:13:39.666 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi174': '172.21.15.174', 'mon.smithi184': '172.21.15.184'} 2024-02-07T12:13:39.666 INFO:tasks.cephadm:Normalizing hostnames... 2024-02-07T12:13:39.666 DEBUG:teuthology.orchestra.run.smithi174:> sudo hostname $(hostname -s) 2024-02-07T12:13:39.693 DEBUG:teuthology.orchestra.run.smithi184:> sudo hostname $(hostname -s) 2024-02-07T12:13:39.723 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-02-07T12:13:39.723 DEBUG:teuthology.orchestra.run.smithi174:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-02-07T12:13:39.992 INFO:teuthology.orchestra.run.smithi174.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Feb 7 12:13 /home/ubuntu/cephtest/cephadm 2024-02-07T12:13:39.992 DEBUG:teuthology.orchestra.run.smithi184:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-02-07T12:13:40.152 INFO:teuthology.orchestra.run.smithi184.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Feb 7 12:13 /home/ubuntu/cephtest/cephadm 2024-02-07T12:13:40.153 DEBUG:teuthology.orchestra.run.smithi174:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-02-07T12:13:40.168 DEBUG:teuthology.orchestra.run.smithi184:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-02-07T12:13:40.192 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-02-07T12:13:40.193 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-02-07T12:13:40.210 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-02-07T12:13:41.053 INFO:teuthology.orchestra.run.smithi174.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-02-07T12:13:41.132 INFO:teuthology.orchestra.run.smithi184.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-02-07T12:13:58.404 INFO:teuthology.orchestra.run.smithi184.stdout:{ 2024-02-07T12:13:58.404 INFO:teuthology.orchestra.run.smithi184.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-02-07T12:13:58.404 INFO:teuthology.orchestra.run.smithi184.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-02-07T12:13:58.404 INFO:teuthology.orchestra.run.smithi184.stdout: "repo_digests": [ 2024-02-07T12:13:58.404 INFO:teuthology.orchestra.run.smithi184.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-02-07T12:13:58.404 INFO:teuthology.orchestra.run.smithi184.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-02-07T12:13:58.404 INFO:teuthology.orchestra.run.smithi184.stdout: ] 2024-02-07T12:13:58.404 INFO:teuthology.orchestra.run.smithi184.stdout:} 2024-02-07T12:14:00.949 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:14:00.949 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-02-07T12:14:00.949 INFO:teuthology.orchestra.run.smithi174.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-02-07T12:14:00.949 INFO:teuthology.orchestra.run.smithi174.stdout: "repo_digests": [ 2024-02-07T12:14:00.949 INFO:teuthology.orchestra.run.smithi174.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-02-07T12:14:00.949 INFO:teuthology.orchestra.run.smithi174.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-02-07T12:14:00.949 INFO:teuthology.orchestra.run.smithi174.stdout: ] 2024-02-07T12:14:00.949 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:14:00.967 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /etc/ceph 2024-02-07T12:14:01.019 DEBUG:teuthology.orchestra.run.smithi184:> sudo mkdir -p /etc/ceph 2024-02-07T12:14:01.054 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 777 /etc/ceph 2024-02-07T12:14:01.080 DEBUG:teuthology.orchestra.run.smithi184:> sudo chmod 777 /etc/ceph 2024-02-07T12:14:01.123 INFO:tasks.cephadm:Writing seed config... 2024-02-07T12:14:01.125 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-02-07T12:14:01.125 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-02-07T12:14:01.125 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-02-07T12:14:01.125 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-02-07T12:14:01.125 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-02-07T12:14:01.126 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-02-07T12:14:01.126 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-02-07T12:14:01.126 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-02-07T12:14:01.127 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:14:01.127 DEBUG:teuthology.orchestra.run.smithi174:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-02-07T12:14:01.141 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 53d92d24-c5b2-11ee-95b6-87774f69a715 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-02-07T12:14:01.142 DEBUG:teuthology.orchestra.run.smithi174:mon.smithi174> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service 2024-02-07T12:14:01.184 INFO:tasks.cephadm:Bootstrapping... 2024-02-07T12:14:01.184 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.174 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:14:01.375 INFO:teuthology.orchestra.run.smithi174.stderr:-------------------------------------------------------------------------------- 2024-02-07T12:14:01.376 INFO:teuthology.orchestra.run.smithi174.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '53d92d24-c5b2-11ee-95b6-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.174', '--skip-admin-label'] 2024-02-07T12:14:01.398 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: 4.9.3-dev 2024-02-07T12:14:01.400 INFO:teuthology.orchestra.run.smithi174.stderr:Verifying podman|docker is present... 2024-02-07T12:14:01.421 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: 4.9.3-dev 2024-02-07T12:14:01.424 INFO:teuthology.orchestra.run.smithi174.stderr:Verifying lvm2 is present... 2024-02-07T12:14:01.424 INFO:teuthology.orchestra.run.smithi174.stderr:Verifying time synchronization is in place... 2024-02-07T12:14:01.431 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-02-07T12:14:01.437 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: inactive 2024-02-07T12:14:01.443 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: enabled 2024-02-07T12:14:01.448 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: active 2024-02-07T12:14:01.449 INFO:teuthology.orchestra.run.smithi174.stderr:Unit chronyd.service is enabled and running 2024-02-07T12:14:01.449 INFO:teuthology.orchestra.run.smithi174.stderr:Repeating the final host check... 2024-02-07T12:14:01.469 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: 4.9.3-dev 2024-02-07T12:14:01.471 INFO:teuthology.orchestra.run.smithi174.stderr:podman (/bin/podman) version 4.9.3 is present 2024-02-07T12:14:01.471 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl is present 2024-02-07T12:14:01.472 INFO:teuthology.orchestra.run.smithi174.stderr:lvcreate is present 2024-02-07T12:14:01.478 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-02-07T12:14:01.483 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: inactive 2024-02-07T12:14:01.490 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: enabled 2024-02-07T12:14:01.495 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: active 2024-02-07T12:14:01.496 INFO:teuthology.orchestra.run.smithi174.stderr:Unit chronyd.service is enabled and running 2024-02-07T12:14:01.496 INFO:teuthology.orchestra.run.smithi174.stderr:Host looks OK 2024-02-07T12:14:01.497 INFO:teuthology.orchestra.run.smithi174.stderr:Cluster fsid: 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:01.497 INFO:teuthology.orchestra.run.smithi174.stderr:Acquiring lock 139624321084960 on /run/cephadm/53d92d24-c5b2-11ee-95b6-87774f69a715.lock 2024-02-07T12:14:01.497 INFO:teuthology.orchestra.run.smithi174.stderr:Lock 139624321084960 acquired on /run/cephadm/53d92d24-c5b2-11ee-95b6-87774f69a715.lock 2024-02-07T12:14:01.497 INFO:teuthology.orchestra.run.smithi174.stderr:Verifying IP 172.21.15.174 port 3300 ... 2024-02-07T12:14:01.497 INFO:teuthology.orchestra.run.smithi174.stderr:Verifying IP 172.21.15.174 port 6789 ... 2024-02-07T12:14:01.498 INFO:teuthology.orchestra.run.smithi174.stderr:Base mon IP is 172.21.15.174, final addrv is [v2:172.21.15.174:3300,v1:172.21.15.174:6789] 2024-02-07T12:14:01.500 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.174 metric 100 2024-02-07T12:14:01.500 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.174 metric 100 2024-02-07T12:14:01.503 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-02-07T12:14:01.503 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-02-07T12:14:01.503 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1761sec hoplimit 64 pref medium 2024-02-07T12:14:01.506 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-02-07T12:14:01.506 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-02-07T12:14:01.506 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-02-07T12:14:01.507 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-02-07T12:14:01.507 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:771b/64 scope link 2024-02-07T12:14:01.507 INFO:teuthology.orchestra.run.smithi174.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-02-07T12:14:01.507 INFO:teuthology.orchestra.run.smithi174.stderr:Mon IP `172.21.15.174` is in CIDR network `172.21.0.0/20` 2024-02-07T12:14:01.507 INFO:teuthology.orchestra.run.smithi174.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-02-07T12:14:01.508 INFO:teuthology.orchestra.run.smithi174.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-02-07T12:14:01.607 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-02-07T12:14:01.959 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Getting image source signatures 2024-02-07T12:14:01.959 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-02-07T12:14:01.959 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-02-07T12:14:01.959 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-02-07T12:14:01.959 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-02-07T12:14:01.959 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-02-07T12:14:01.959 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-02-07T12:14:02.086 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: Writing manifest to image destination 2024-02-07T12:14:02.188 INFO:teuthology.orchestra.run.smithi174.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-02-07T12:14:02.596 INFO:teuthology.orchestra.run.smithi174.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-02-07T12:14:03.078 INFO:teuthology.orchestra.run.smithi174.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-02-07T12:14:03.079 INFO:teuthology.orchestra.run.smithi174.stderr:Extracting ceph user uid/gid from container image... 2024-02-07T12:14:03.470 INFO:teuthology.orchestra.run.smithi174.stderr:stat: 167 167 2024-02-07T12:14:04.067 INFO:teuthology.orchestra.run.smithi174.stderr:Creating initial keys... 2024-02-07T12:14:04.464 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph-authtool: AQAMdMNlJmZ4FhAAtVsaGoSbTOo93puC+pPLBA== 2024-02-07T12:14:05.319 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph-authtool: AQANdMNlX/xEDhAAZTogTaAeAF6JPbeigpWwrg== 2024-02-07T12:14:06.207 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph-authtool: AQAOdMNlvmWuCBAAkzRM4OdjU5/KjOBSJI0Kuw== 2024-02-07T12:14:06.831 INFO:teuthology.orchestra.run.smithi174.stderr:Creating initial monmap... 2024-02-07T12:14:09.573 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-02-07T12:14:09.573 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-02-07T12:14:09.574 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:09.574 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-02-07T12:14:15.315 INFO:teuthology.orchestra.run.smithi174.stderr:monmaptool for smithi174 [v2:172.21.15.174:3300,v1:172.21.15.174:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-02-07T12:14:15.315 INFO:teuthology.orchestra.run.smithi174.stderr:setting min_mon_release = octopus 2024-02-07T12:14:15.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/monmaptool: set fsid to 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:15.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-02-07T12:14:15.315 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:14:15.315 INFO:teuthology.orchestra.run.smithi174.stderr:Creating mon... 2024-02-07T12:14:18.200 INFO:teuthology.orchestra.run.smithi174.stderr:create mon.smithi174 on 2024-02-07T12:14:18.405 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-02-07T12:14:18.549 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715.target → /etc/systemd/system/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715.target. 2024-02-07T12:14:18.550 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715.target → /etc/systemd/system/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715.target. 2024-02-07T12:14:18.840 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Failed to reset failed state of unit ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service: Unit ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service not loaded. 2024-02-07T12:14:18.847 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Created symlink /etc/systemd/system/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715.target.wants/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service → /etc/systemd/system/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@.service. 2024-02-07T12:14:19.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:19 smithi174 podman[24200]: 2024-02-07 12:14:19.443268603 +0000 UTC m=+0.323476456 container start 5f0852101e14658aa91879cd1673454921e83ad1bb367f6c040f57e50798dd8e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, distribution-scope=public, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux ) 2024-02-07T12:14:19.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:19 smithi174 ceph-mon[24219]: mkfs 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:19.536 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: disabled 2024-02-07T12:14:19.543 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: inactive 2024-02-07T12:14:19.543 INFO:teuthology.orchestra.run.smithi174.stderr:firewalld.service is not enabled 2024-02-07T12:14:19.544 INFO:teuthology.orchestra.run.smithi174.stderr:Not possible to enable service . firewalld.service is not available 2024-02-07T12:14:19.544 INFO:teuthology.orchestra.run.smithi174.stderr:Waiting for mon to start... 2024-02-07T12:14:19.544 INFO:teuthology.orchestra.run.smithi174.stderr:Waiting for mon... 2024-02-07T12:14:19.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:19 smithi174 ceph-mon[24219]: mon.smithi174 is new leader, mons smithi174 in quorum (ranks 0) 2024-02-07T12:14:19.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:19 smithi174 bash[24200]: 5f0852101e14658aa91879cd1673454921e83ad1bb367f6c040f57e50798dd8e 2024-02-07T12:14:19.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:19 smithi174 systemd[1]: Started Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:14:20.090 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: cluster: 2024-02-07T12:14:20.090 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: id: 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:20.090 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-02-07T12:14:20.090 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:20.090 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: services: 2024-02-07T12:14:20.090 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi174 (age 0.583237s) 2024-02-07T12:14:20.090 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mgr: no daemons active 2024-02-07T12:14:20.091 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-02-07T12:14:20.091 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:20.091 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: data: 2024-02-07T12:14:20.091 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-02-07T12:14:20.091 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-02-07T12:14:20.091 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-02-07T12:14:20.091 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: pgs: 2024-02-07T12:14:20.091 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:20.648 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:20 smithi174 ceph-mon[24219]: mon.smithi174 is new leader, mons smithi174 in quorum (ranks 0) 2024-02-07T12:14:20.648 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:20 smithi174 ceph-mon[24219]: monmap e1: 1 mons at {smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0]} 2024-02-07T12:14:20.649 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:20 smithi174 ceph-mon[24219]: fsmap 2024-02-07T12:14:20.649 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:20 smithi174 ceph-mon[24219]: osdmap e1: 0 total, 0 up, 0 in 2024-02-07T12:14:20.649 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:20 smithi174 ceph-mon[24219]: mgrmap e1: no daemons active 2024-02-07T12:14:20.649 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:20 smithi174 ceph-mon[24219]: from='client.? 172.21.15.174:0/2185134322' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-02-07T12:14:20.651 INFO:teuthology.orchestra.run.smithi174.stderr:mon is available 2024-02-07T12:14:20.651 INFO:teuthology.orchestra.run.smithi174.stderr:Assimilating anything we can from ceph.conf... 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: [global] 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: fsid = 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.174:3300,v1:172.21.15.174:6789] 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: [mgr] 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-02-07T12:14:21.921 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:21.922 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: [osd] 2024-02-07T12:14:21.922 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-02-07T12:14:21.922 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-02-07T12:14:22.221 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:21 smithi174 ceph-mon[24219]: from='client.? 172.21.15.174:0/3902537289' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-02-07T12:14:22.221 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:21 smithi174 ceph-mon[24219]: from='client.? 172.21.15.174:0/3902537289' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-02-07T12:14:22.499 INFO:teuthology.orchestra.run.smithi174.stderr:Generating new minimal ceph.conf... 2024-02-07T12:14:23.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:23 smithi174 ceph-mon[24219]: from='client.? 172.21.15.174:0/3578838634' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:14:24.570 INFO:teuthology.orchestra.run.smithi174.stderr:Restarting the monitor... 2024-02-07T12:14:24.993 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:24 smithi174 systemd[1]: Stopping Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:14:24.993 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:24 smithi174 bash[24451]: Error: no container with name or ID "ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon.smithi174" found: no such container 2024-02-07T12:14:24.994 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:24 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[24215]: 2024-02-07T12:14:24.734+0000 7fcd4a929700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi174 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:14:24.994 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:24 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[24215]: 2024-02-07T12:14:24.734+0000 7fcd4a929700 -1 mon.smithi174@0(leader) e1 *** Got Signal Terminated *** 2024-02-07T12:14:25.253 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:24 smithi174 podman[24465]: 2024-02-07 12:14:24.991711929 +0000 UTC m=+0.363532822 container died 5f0852101e14658aa91879cd1673454921e83ad1bb367f6c040f57e50798dd8e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, distribution-scope=public, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vcs-type=git, RELEASE=HEAD, GIT_CLEAN=True, ceph=True, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-02-07T12:14:25.512 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 podman[24465]: 2024-02-07 12:14:25.250935332 +0000 UTC m=+0.622756232 container cleanup 5f0852101e14658aa91879cd1673454921e83ad1bb367f6c040f57e50798dd8e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, distribution-scope=public, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, version=8, maintainer=Guillaume Abrioux ) 2024-02-07T12:14:25.512 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 bash[24465]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174 2024-02-07T12:14:25.512 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 bash[24491]: Error: no container with name or ID "ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon.smithi174" found: no such container 2024-02-07T12:14:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 podman[24479]: 2024-02-07 12:14:25.510061784 +0000 UTC m=+0.521088847 container remove 5f0852101e14658aa91879cd1673454921e83ad1bb367f6c040f57e50798dd8e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, maintainer=Guillaume Abrioux , architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, ceph=True, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0) 2024-02-07T12:14:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service: Deactivated successfully. 2024-02-07T12:14:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 systemd[1]: Stopped Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:14:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 systemd[1]: Starting Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:14:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 podman[24560]: 2024-02-07 12:14:25.651028513 +0000 UTC m=+0.017982031 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-02-07T12:14:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 podman[24560]: 2024-02-07 12:14:25.752372837 +0000 UTC m=+0.119326347 container create 812c56ca3a759b56c90acc06dbeca4eefe5b5765b36b7955403bfa2beb5f3a24 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-02-07T12:14:26.043 INFO:teuthology.orchestra.run.smithi174.stderr:Setting mon public_network to 172.21.0.0/20 2024-02-07T12:14:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 podman[24560]: 2024-02-07 12:14:25.954699143 +0000 UTC m=+0.321652659 container init 812c56ca3a759b56c90acc06dbeca4eefe5b5765b36b7955403bfa2beb5f3a24 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, version=8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True) 2024-02-07T12:14:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 podman[24560]: 2024-02-07 12:14:25.957535345 +0000 UTC m=+0.324488858 container start 812c56ca3a759b56c90acc06dbeca4eefe5b5765b36b7955403bfa2beb5f3a24 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.buildah.version=1.19.8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-02-07T12:14:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:14:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-02-07T12:14:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: pidfile_write: ignore empty --pid-file 2024-02-07T12:14:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: load: jerasure load: lrc 2024-02-07T12:14:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: RocksDB version: 6.15.5 2024-02-07T12:14:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Compile date Apr 18 2022 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: DB SUMMARY 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: DB Session ID: 6DAGMV18UJ3P5FCEVRKJ 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi174/store.db dir, Total Num: 1, files: 000008.sst 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi174/store.db: 000010.log size: 83242 ; 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:14:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.env: 0x56125e2c6860 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.fs: Posix File System 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.info_log: 0x56125ee37fa0 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.statistics: (nil) 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.use_fsync: 0 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:14:26.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.db_log_dir: 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi174/store.db 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:14:26.170 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.write_buffer_manager: 0x56125ef282d0 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:14:26.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.unordered_write: 0 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.row_cache: None 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.wal_filter: None 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.preserve_deletes: 0 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:14:26.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:14:26.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_open_files: -1 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Compression algorithms supported: 2024-02-07T12:14:26.175 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: kZSTD supported: 0 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi174/store.db/MANIFEST-000009 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.merge_operator: 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_filter: None 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:14:26.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56125ee03d00) 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_type: 0 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_block_index_type: 0 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_shortening: 1 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: hash_index_allow_collision: 1 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: checksum: 1 2024-02-07T12:14:26.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: no_block_cache: 0 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache: 0x56125ee6e170 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_options: 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: capacity : 536870912 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: num_shard_bits : 4 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: strict_capacity_limit : 0 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_compressed: (nil) 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: persistent_cache: (nil) 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_size: 4096 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_size_deviation: 10 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_restart_interval: 16 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_block_restart_interval: 1 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: metadata_block_size: 4096 2024-02-07T12:14:26.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: partition_filters: 0 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: use_delta_encoding: 1 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: whole_key_filtering: 1 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: verify_compression: 0 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: format_version: 4 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: enable_index_compression: 1 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_align: 0 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compression: NoCompression 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:14:26.179 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.num_levels: 7 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:14:26.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:14:26.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.arena_block_size: 4194304 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:14:26.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:14:26.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.ttl: 2592000 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:14:26.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi174/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308065982867, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308065983696, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79680, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77944, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9900, "raw_average_key_size": 47, "raw_value_size": 72240, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 210, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1707308065, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1fd786af-80fc-4375-bba5-16f10eb5cbca", "db_session_id": "6DAGMV18UJ3P5FCEVRKJ"}} 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308065984042, "job": 1, "event": "recovery_finished"} 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi174/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x56125ee54a80 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: DB pointer 0x56125ee64000 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-02-07T12:14:26.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** DB Stats ** 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: L0 2/0 79.61 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.5 0.00 0.00 1 0.000 0 0 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Sum 2/0 79.61 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.5 0.00 0.00 1 0.000 0 0 2024-02-07T12:14:26.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.5 0.00 0.00 1 0.000 0 0 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 164.5 0.00 0.00 1 0.000 0 0 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative compaction: 0.00 GB write, 25.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval compaction: 0.00 GB write, 25.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:14:26.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: L0 2/0 79.61 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.5 0.00 0.00 1 0.000 0 0 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Sum 2/0 79.61 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.5 0.00 0.00 1 0.000 0 0 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:14:26.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 164.5 0.00 0.00 1 0.000 0 0 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative compaction: 0.00 GB write, 25.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: starting mon.smithi174 rank 0 at public addrs [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] at bind addrs [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi174 fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:26.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: mon.smithi174@-1(???) e1 preinit fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: mon.smithi174@-1(???).mds e1 new map 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: mon.smithi174@-1(???).mds e1 print_map 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: e1 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: legacy client fscid: -1 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: No filesystems configured 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: mon.smithi174@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: mon.smithi174@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: mon.smithi174@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: mon.smithi174@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:14:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: mon.smithi174@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta expand map: {default=false} 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta from 'false' to 'false' 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta expanded map: {default=false} 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta expand map: {default=info} 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta from 'info' to 'info' 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta expanded map: {default=info} 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta expand map: {default=daemon} 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta from 'daemon' to 'daemon' 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta expanded map: {default=daemon} 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta expand map: {default=debug} 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta from 'debug' to 'debug' 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:25 smithi174 ceph-mon[24580]: expand_channel_meta expanded map: {default=debug} 2024-02-07T12:14:26.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:26 smithi174 bash[24560]: 812c56ca3a759b56c90acc06dbeca4eefe5b5765b36b7955403bfa2beb5f3a24 2024-02-07T12:14:26.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:26 smithi174 systemd[1]: Started Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:14:26.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:26 smithi174 ceph-mon[24580]: mon.smithi174 is new leader, mons smithi174 in quorum (ranks 0) 2024-02-07T12:14:26.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:26 smithi174 ceph-mon[24580]: monmap e1: 1 mons at {smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0]} 2024-02-07T12:14:26.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:26 smithi174 ceph-mon[24580]: fsmap 2024-02-07T12:14:26.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:26 smithi174 ceph-mon[24580]: osdmap e1: 0 total, 0 up, 0 in 2024-02-07T12:14:26.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:26 smithi174 ceph-mon[24580]: mgrmap e1: no daemons active 2024-02-07T12:14:27.610 INFO:teuthology.orchestra.run.smithi174.stderr:Wrote config to /etc/ceph/ceph.conf 2024-02-07T12:14:27.610 INFO:teuthology.orchestra.run.smithi174.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:14:27.611 INFO:teuthology.orchestra.run.smithi174.stderr:Creating mgr... 2024-02-07T12:14:27.611 INFO:teuthology.orchestra.run.smithi174.stderr:Verifying port 9283 ... 2024-02-07T12:14:27.795 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Failed to reset failed state of unit ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mgr.smithi174.whsozo.service: Unit ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mgr.smithi174.whsozo.service not loaded. 2024-02-07T12:14:27.802 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: Created symlink /etc/systemd/system/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715.target.wants/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mgr.smithi174.whsozo.service → /etc/systemd/system/ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@.service. 2024-02-07T12:14:27.970 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:27 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2735590179' entity='client.admin' 2024-02-07T12:14:28.450 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: disabled 2024-02-07T12:14:28.459 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: inactive 2024-02-07T12:14:28.459 INFO:teuthology.orchestra.run.smithi174.stderr:firewalld.service is not enabled 2024-02-07T12:14:28.459 INFO:teuthology.orchestra.run.smithi174.stderr:Not possible to enable service . firewalld.service is not available 2024-02-07T12:14:28.469 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: disabled 2024-02-07T12:14:28.478 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: inactive 2024-02-07T12:14:28.479 INFO:teuthology.orchestra.run.smithi174.stderr:firewalld.service is not enabled 2024-02-07T12:14:28.479 INFO:teuthology.orchestra.run.smithi174.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-02-07T12:14:28.479 INFO:teuthology.orchestra.run.smithi174.stderr:Waiting for mgr to start... 2024-02-07T12:14:28.479 INFO:teuthology.orchestra.run.smithi174.stderr:Waiting for mgr... 2024-02-07T12:14:28.987 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: { 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "fsid": "53d92d24-c5b2-11ee-95b6-87774f69a715", 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "health": { 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "checks": {}, 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "mutes": [] 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum": [ 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 0 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:28.988 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "smithi174" 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "monmap": { 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osdmap": { 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-07T12:14:28.989 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "pgmap": { 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "fsmap": { 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-07T12:14:28.990 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "available": false, 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "modules": [ 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "iostat", 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "nfs", 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "restful" 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "servicemap": { 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "modified": "2024-02-07T12:14:19.506995+0000", 2024-02-07T12:14:28.991 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:14:28.992 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:28.992 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-07T12:14:28.992 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: } 2024-02-07T12:14:29.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:29 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3823582946' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-07T12:14:29.610 INFO:teuthology.orchestra.run.smithi174.stderr:mgr not available, waiting (1/15)... 2024-02-07T12:14:32.168 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: { 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "fsid": "53d92d24-c5b2-11ee-95b6-87774f69a715", 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "health": { 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "checks": {}, 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "mutes": [] 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum": [ 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 0 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-07T12:14:32.169 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "smithi174" 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "monmap": { 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osdmap": { 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-07T12:14:32.170 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "pgmap": { 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "fsmap": { 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-07T12:14:32.171 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "available": false, 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "modules": [ 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "iostat", 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "nfs", 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "restful" 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "servicemap": { 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "modified": "2024-02-07T12:14:19.506995+0000", 2024-02-07T12:14:32.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:14:32.173 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:32.173 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-07T12:14:32.173 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: } 2024-02-07T12:14:32.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:32 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2394480692' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-07T12:14:32.799 INFO:teuthology.orchestra.run.smithi174.stderr:mgr not available, waiting (2/15)... 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: Activating manager daemon smithi174.whsozo 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: mgrmap e2: smithi174.whsozo(active, starting, since 0.00281618s) 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:14:34.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:14:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:14:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:34 smithi174 ceph-mon[24580]: from='mgr.14100 172.21.15.174:0/3720821757' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:35.307 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: { 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "fsid": "53d92d24-c5b2-11ee-95b6-87774f69a715", 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "health": { 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "checks": {}, 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "mutes": [] 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum": [ 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 0 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum_names": [ 2024-02-07T12:14:35.308 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "smithi174" 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "monmap": { 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_mons": 1 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osdmap": { 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_osds": 0, 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-02-07T12:14:35.309 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "pgmap": { 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_pools": 0, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_objects": 0, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-02-07T12:14:35.313 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "bytes_total": 0 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "fsmap": { 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "by_rank": [], 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "up:standby": 0 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "mgrmap": { 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "available": true, 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "modules": [ 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "iostat", 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "nfs", 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "restful" 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ], 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:14:35.314 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:35.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "servicemap": { 2024-02-07T12:14:35.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 1, 2024-02-07T12:14:35.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "modified": "2024-02-07T12:14:19.506995+0000", 2024-02-07T12:14:35.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "services": {} 2024-02-07T12:14:35.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: }, 2024-02-07T12:14:35.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "progress_events": {} 2024-02-07T12:14:35.315 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: } 2024-02-07T12:14:35.839 INFO:teuthology.orchestra.run.smithi174.stderr:mgr is available 2024-02-07T12:14:36.480 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:36 smithi174 ceph-mon[24580]: mgrmap e3: smithi174.whsozo(active, since 1.00678s) 2024-02-07T12:14:36.481 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:36 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2265760545' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-02-07T12:14:36.482 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:36.482 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: [global] 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: fsid = 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: [mgr] 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: [osd] 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-02-07T12:14:36.483 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-02-07T12:14:37.112 INFO:teuthology.orchestra.run.smithi174.stderr:Enabling cephadm module... 2024-02-07T12:14:37.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:37 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/816609039' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-02-07T12:14:38.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:38 smithi174 ceph-mon[24580]: mgrmap e4: smithi174.whsozo(active, since 3s) 2024-02-07T12:14:38.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:38 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3243665321' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-02-07T12:14:39.465 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:39 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3243665321' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-02-07T12:14:39.466 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:39 smithi174 ceph-mon[24580]: mgrmap e5: smithi174.whsozo(active, since 4s) 2024-02-07T12:14:39.554 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: { 2024-02-07T12:14:39.554 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 5, 2024-02-07T12:14:39.554 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "available": true, 2024-02-07T12:14:39.555 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "active_name": "smithi174.whsozo", 2024-02-07T12:14:39.555 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_standby": 0 2024-02-07T12:14:39.555 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: } 2024-02-07T12:14:40.114 INFO:teuthology.orchestra.run.smithi174.stderr:Waiting for the mgr to restart... 2024-02-07T12:14:40.114 INFO:teuthology.orchestra.run.smithi174.stderr:Waiting for mgr epoch 5... 2024-02-07T12:14:40.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:40 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3227136839' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-02-07T12:14:45.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: { 2024-02-07T12:14:45.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-02-07T12:14:45.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "initialized": true 2024-02-07T12:14:45.172 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: } 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: Active manager daemon smithi174.whsozo restarted 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: Activating manager daemon smithi174.whsozo 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: osdmap e2: 0 total, 0 up, 0 in 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: mgrmap e6: smithi174.whsozo(active, starting, since 0.0527513s) 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:14:45.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:14:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:14:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:14:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:14:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:14:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:45 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:14:45.811 INFO:teuthology.orchestra.run.smithi174.stderr:mgr epoch 5 is available 2024-02-07T12:14:45.811 INFO:teuthology.orchestra.run.smithi174.stderr:Setting orchestrator backend to cephadm... 2024-02-07T12:14:46.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:46 smithi174 ceph-mon[24580]: [07/Feb/2024:12:14:45] ENGINE Bus STARTING 2024-02-07T12:14:46.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:46 smithi174 ceph-mon[24580]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-02-07T12:14:46.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:46 smithi174 ceph-mon[24580]: mgrmap e7: smithi174.whsozo(active, since 1.05512s) 2024-02-07T12:14:46.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:46 smithi174 ceph-mon[24580]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-02-07T12:14:46.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:46 smithi174 ceph-mon[24580]: [07/Feb/2024:12:14:45] ENGINE Serving on https://172.21.15.174:7150 2024-02-07T12:14:46.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:46 smithi174 ceph-mon[24580]: [07/Feb/2024:12:14:45] ENGINE Bus STARTED 2024-02-07T12:14:46.442 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:46 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:46.442 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:46 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:14:47.625 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:47 smithi174 ceph-mon[24580]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:14:47.625 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:47 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:47.625 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:47 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:14:47.625 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:47 smithi174 ceph-mon[24580]: mgrmap e8: smithi174.whsozo(active, since 2s) 2024-02-07T12:14:47.625 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: value unchanged 2024-02-07T12:14:48.206 INFO:teuthology.orchestra.run.smithi174.stderr:Generating ssh key... 2024-02-07T12:14:48.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:48 smithi174 ceph-mon[24580]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:14:49.709 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:49 smithi174 ceph-mon[24580]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:14:49.709 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:49 smithi174 ceph-mon[24580]: Generating ssh key... 2024-02-07T12:14:49.709 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:49 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:49.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:49 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:50.596 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDBarTRTYp8QX+w0OGelEsd0ItvB1tEBc3Qrzn2Ec55HDhReXVvNORCT50/gca8iAHuH19ELRAUSMZo0t8qq5gdWa6paqtSoVuGX1GV770wZwoheHdRGldR9RP9ZBA87snuTDP+ESK4+LhU91gNt1vofZKiUTMWaEU2UwlQz3TbKQlzzk+XdvyqbiZNmtaa5kt6OdUQejBhDA9KeFyzfLjhLPpXRFLdyIqCbm6Gw9jPEY/3+5rlqcCV1BaQS/OVfNAz6AP+WmBRfMaGkuOFKHntqYZmVk479FpOE8Yh3AQMZ1hfJEXSm1MBm0LjrAXPUBg86c4eOAJlaFUFXLae4CjfAxQtW/k/tp0VdFe+txap90CEOtONDuJ/Of7Yn/F6cIwtXe7qCovTkE5PrGUTRD3sGZmgIrmWnaSQeBUNUwu71eLY08aN8h+FQgWHhJfAHEfxjX0JJafl6cdIVSdb6xCqYKZ70cjrP1g1s15KuxAO86r4yZ1c+iXJXkf/6GJSg+0= ceph-53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:14:51.176 INFO:teuthology.orchestra.run.smithi174.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-02-07T12:14:51.177 INFO:teuthology.orchestra.run.smithi174.stderr:Adding key to root@localhost authorized_keys... 2024-02-07T12:14:51.177 INFO:teuthology.orchestra.run.smithi174.stderr:Adding host smithi174... 2024-02-07T12:14:52.110 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:51 smithi174 ceph-mon[24580]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:14:52.793 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Added host 'smithi174' with addr '172.21.15.174' 2024-02-07T12:14:53.114 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:52 smithi174 ceph-mon[24580]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi174", "addr": "172.21.15.174", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:14:53.114 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:52 smithi174 ceph-mon[24580]: Deploying cephadm binary to smithi174 2024-02-07T12:14:53.114 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:52 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:53.114 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:52 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:14:53.416 INFO:teuthology.orchestra.run.smithi174.stderr:Deploying mon service with default placement... 2024-02-07T12:14:54.047 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Scheduled mon update... 2024-02-07T12:14:54.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:53 smithi174 ceph-mon[24580]: Added host smithi174 2024-02-07T12:14:54.681 INFO:teuthology.orchestra.run.smithi174.stderr:Deploying mgr service with default placement... 2024-02-07T12:14:55.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:55 smithi174 ceph-mon[24580]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:14:55.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:55 smithi174 ceph-mon[24580]: Saving service mon spec with placement count:5 2024-02-07T12:14:55.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:55 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:55.503 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-02-07T12:14:56.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:56 smithi174 ceph-mon[24580]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:14:56.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:56 smithi174 ceph-mon[24580]: Saving service mgr spec with placement count:2 2024-02-07T12:14:56.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:56 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:56.944 INFO:teuthology.orchestra.run.smithi174.stderr:Deploying crash service with default placement... 2024-02-07T12:14:57.638 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Scheduled crash update... 2024-02-07T12:14:57.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:57 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:57.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:57 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:58.310 INFO:teuthology.orchestra.run.smithi174.stderr:Deploying prometheus service with default placement... 2024-02-07T12:14:58.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:58 smithi174 ceph-mon[24580]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:14:58.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:58 smithi174 ceph-mon[24580]: Saving service crash spec with placement * 2024-02-07T12:14:58.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:58 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:14:58.916 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-02-07T12:14:59.893 INFO:teuthology.orchestra.run.smithi174.stderr:Deploying grafana service with default placement... 2024-02-07T12:15:00.145 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:59 smithi174 ceph-mon[24580]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:00.145 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:59 smithi174 ceph-mon[24580]: Saving service prometheus spec with placement count:1 2024-02-07T12:15:00.145 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:59 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:00.145 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:14:59 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:00.602 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-02-07T12:15:01.198 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:00 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:01.199 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:00 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:01.201 INFO:teuthology.orchestra.run.smithi174.stderr:Deploying node-exporter service with default placement... 2024-02-07T12:15:01.848 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-02-07T12:15:02.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:01 smithi174 ceph-mon[24580]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:02.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:01 smithi174 ceph-mon[24580]: Saving service grafana spec with placement count:1 2024-02-07T12:15:02.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:01 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:02.618 INFO:teuthology.orchestra.run.smithi174.stderr:Deploying alertmanager service with default placement... 2024-02-07T12:15:03.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:02 smithi174 ceph-mon[24580]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:03.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:03 smithi174 ceph-mon[24580]: Saving service node-exporter spec with placement * 2024-02-07T12:15:03.740 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-02-07T12:15:05.044 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:04 smithi174 ceph-mon[24580]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:05.045 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:04 smithi174 ceph-mon[24580]: Saving service alertmanager spec with placement count:1 2024-02-07T12:15:05.045 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:04 smithi174 ceph-mon[24580]: from='mgr.14118 172.21.15.174:0/793528880' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:05.045 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:04 smithi174 ceph-mon[24580]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:06.141 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:06 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/4042392945' entity='client.admin' 2024-02-07T12:15:07.023 INFO:teuthology.orchestra.run.smithi174.stderr:Enabling the dashboard module... 2024-02-07T12:15:07.280 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:07 smithi174 ceph-mon[24580]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:07.280 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:07 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/4088327515' entity='client.admin' 2024-02-07T12:15:08.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:08 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3092372815' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-02-07T12:15:09.233 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:09 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3092372815' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-02-07T12:15:09.233 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:09 smithi174 ceph-mon[24580]: mgrmap e9: smithi174.whsozo(active, since 23s) 2024-02-07T12:15:09.287 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: { 2024-02-07T12:15:09.288 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "epoch": 9, 2024-02-07T12:15:09.288 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "available": true, 2024-02-07T12:15:09.288 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "active_name": "smithi174.whsozo", 2024-02-07T12:15:09.288 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "num_standby": 0 2024-02-07T12:15:09.288 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: } 2024-02-07T12:15:09.977 INFO:teuthology.orchestra.run.smithi174.stderr:Waiting for the mgr to restart... 2024-02-07T12:15:09.977 INFO:teuthology.orchestra.run.smithi174.stderr:Waiting for mgr epoch 9... 2024-02-07T12:15:10.086 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:10 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/1027651494' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-02-07T12:15:14.946 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: { 2024-02-07T12:15:14.947 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-02-07T12:15:14.947 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: "initialized": true 2024-02-07T12:15:14.947 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: } 2024-02-07T12:15:15.135 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: Active manager daemon smithi174.whsozo restarted 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: Activating manager daemon smithi174.whsozo 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: osdmap e3: 0 total, 0 up, 0 in 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: mgrmap e10: smithi174.whsozo(active, starting, since 0.0551044s) 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:15:15.136 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:15:15.137 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:14 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:15.837 INFO:teuthology.orchestra.run.smithi174.stderr:mgr epoch 9 is available 2024-02-07T12:15:15.837 INFO:teuthology.orchestra.run.smithi174.stderr:Generating a dashboard self-signed certificate... 2024-02-07T12:15:15.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:15 smithi174 ceph-mon[24580]: [07/Feb/2024:12:15:14] ENGINE Bus STARTING 2024-02-07T12:15:15.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:15 smithi174 ceph-mon[24580]: [07/Feb/2024:12:15:14] ENGINE Serving on https://172.21.15.174:7150 2024-02-07T12:15:15.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:15 smithi174 ceph-mon[24580]: [07/Feb/2024:12:15:14] ENGINE Bus STARTED 2024-02-07T12:15:15.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:15 smithi174 ceph-mon[24580]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-02-07T12:15:15.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:15 smithi174 ceph-mon[24580]: mgrmap e11: smithi174.whsozo(active, since 1.05785s) 2024-02-07T12:15:15.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:15 smithi174 ceph-mon[24580]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-02-07T12:15:15.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:15 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:16.580 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: Self-signed certificate created 2024-02-07T12:15:17.193 INFO:teuthology.orchestra.run.smithi174.stderr:Creating initial admin user... 2024-02-07T12:15:17.578 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:17 smithi174 ceph-mon[24580]: mgrmap e12: smithi174.whsozo(active, since 2s) 2024-02-07T12:15:17.578 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:17 smithi174 ceph-mon[24580]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:17.578 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:17 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:17.578 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:17 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:18.121 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$Jk7lqECfv8k5zVBkXyTjBebZ8jxn0GtlA9EPluwanKH6sPgFq44Ba", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1707308118, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-02-07T12:15:19.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:18 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:19.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:18 smithi174 ceph-mon[24580]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:19.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:18 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:19.856 INFO:teuthology.orchestra.run.smithi174.stderr:Fetching dashboard port number... 2024-02-07T12:15:20.505 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: 8443 2024-02-07T12:15:21.121 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: disabled 2024-02-07T12:15:21.128 INFO:teuthology.orchestra.run.smithi174.stderr:systemctl: inactive 2024-02-07T12:15:21.128 INFO:teuthology.orchestra.run.smithi174.stderr:firewalld.service is not enabled 2024-02-07T12:15:21.128 INFO:teuthology.orchestra.run.smithi174.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-02-07T12:15:21.131 INFO:teuthology.orchestra.run.smithi174.stderr:Ceph Dashboard is now available at: 2024-02-07T12:15:21.131 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:15:21.131 INFO:teuthology.orchestra.run.smithi174.stderr: URL: https://smithi174.front.sepia.ceph.com:8443/ 2024-02-07T12:15:21.131 INFO:teuthology.orchestra.run.smithi174.stderr: User: admin 2024-02-07T12:15:21.131 INFO:teuthology.orchestra.run.smithi174.stderr: Password: gtungol47y 2024-02-07T12:15:21.131 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:15:21.131 INFO:teuthology.orchestra.run.smithi174.stderr:Enabling autotune for osd_memory_target 2024-02-07T12:15:21.252 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:20 smithi174 ceph-mon[24580]: mgrmap e13: smithi174.whsozo(active, since 6s) 2024-02-07T12:15:21.253 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:20 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/374461659' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-02-07T12:15:23.084 INFO:teuthology.orchestra.run.smithi174.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-02-07T12:15:23.775 INFO:teuthology.orchestra.run.smithi174.stderr:You can access the Ceph CLI with: 2024-02-07T12:15:23.775 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:15:23.775 INFO:teuthology.orchestra.run.smithi174.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:15:23.775 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:15:23.775 INFO:teuthology.orchestra.run.smithi174.stderr:Please consider enabling telemetry to help improve Ceph: 2024-02-07T12:15:23.776 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:15:23.776 INFO:teuthology.orchestra.run.smithi174.stderr: ceph telemetry on 2024-02-07T12:15:23.776 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:15:23.776 INFO:teuthology.orchestra.run.smithi174.stderr:For more information see: 2024-02-07T12:15:23.776 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:15:23.776 INFO:teuthology.orchestra.run.smithi174.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-02-07T12:15:23.776 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:15:23.776 INFO:teuthology.orchestra.run.smithi174.stderr:Bootstrap complete. 2024-02-07T12:15:23.812 INFO:tasks.cephadm:Fetching config... 2024-02-07T12:15:23.812 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:15:23.812 DEBUG:teuthology.orchestra.run.smithi174:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-02-07T12:15:23.828 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-02-07T12:15:23.828 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:15:23.828 DEBUG:teuthology.orchestra.run.smithi174:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-02-07T12:15:23.883 INFO:tasks.cephadm:Fetching mon keyring... 2024-02-07T12:15:23.884 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:15:23.884 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/keyring of=/dev/stdout 2024-02-07T12:15:23.950 INFO:tasks.cephadm:Fetching pub ssh key... 2024-02-07T12:15:23.951 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:15:23.951 DEBUG:teuthology.orchestra.run.smithi174:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-02-07T12:15:24.005 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-02-07T12:15:24.006 DEBUG:teuthology.orchestra.run.smithi174:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDBarTRTYp8QX+w0OGelEsd0ItvB1tEBc3Qrzn2Ec55HDhReXVvNORCT50/gca8iAHuH19ELRAUSMZo0t8qq5gdWa6paqtSoVuGX1GV770wZwoheHdRGldR9RP9ZBA87snuTDP+ESK4+LhU91gNt1vofZKiUTMWaEU2UwlQz3TbKQlzzk+XdvyqbiZNmtaa5kt6OdUQejBhDA9KeFyzfLjhLPpXRFLdyIqCbm6Gw9jPEY/3+5rlqcCV1BaQS/OVfNAz6AP+WmBRfMaGkuOFKHntqYZmVk479FpOE8Yh3AQMZ1hfJEXSm1MBm0LjrAXPUBg86c4eOAJlaFUFXLae4CjfAxQtW/k/tp0VdFe+txap90CEOtONDuJ/Of7Yn/F6cIwtXe7qCovTkE5PrGUTRD3sGZmgIrmWnaSQeBUNUwu71eLY08aN8h+FQgWHhJfAHEfxjX0JJafl6cdIVSdb6xCqYKZ70cjrP1g1s15KuxAO86r4yZ1c+iXJXkf/6GJSg+0= ceph-53d92d24-c5b2-11ee-95b6-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-02-07T12:15:24.083 INFO:teuthology.orchestra.run.smithi174.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDBarTRTYp8QX+w0OGelEsd0ItvB1tEBc3Qrzn2Ec55HDhReXVvNORCT50/gca8iAHuH19ELRAUSMZo0t8qq5gdWa6paqtSoVuGX1GV770wZwoheHdRGldR9RP9ZBA87snuTDP+ESK4+LhU91gNt1vofZKiUTMWaEU2UwlQz3TbKQlzzk+XdvyqbiZNmtaa5kt6OdUQejBhDA9KeFyzfLjhLPpXRFLdyIqCbm6Gw9jPEY/3+5rlqcCV1BaQS/OVfNAz6AP+WmBRfMaGkuOFKHntqYZmVk479FpOE8Yh3AQMZ1hfJEXSm1MBm0LjrAXPUBg86c4eOAJlaFUFXLae4CjfAxQtW/k/tp0VdFe+txap90CEOtONDuJ/Of7Yn/F6cIwtXe7qCovTkE5PrGUTRD3sGZmgIrmWnaSQeBUNUwu71eLY08aN8h+FQgWHhJfAHEfxjX0JJafl6cdIVSdb6xCqYKZ70cjrP1g1s15KuxAO86r4yZ1c+iXJXkf/6GJSg+0= ceph-53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:15:24.112 DEBUG:teuthology.orchestra.run.smithi184:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDBarTRTYp8QX+w0OGelEsd0ItvB1tEBc3Qrzn2Ec55HDhReXVvNORCT50/gca8iAHuH19ELRAUSMZo0t8qq5gdWa6paqtSoVuGX1GV770wZwoheHdRGldR9RP9ZBA87snuTDP+ESK4+LhU91gNt1vofZKiUTMWaEU2UwlQz3TbKQlzzk+XdvyqbiZNmtaa5kt6OdUQejBhDA9KeFyzfLjhLPpXRFLdyIqCbm6Gw9jPEY/3+5rlqcCV1BaQS/OVfNAz6AP+WmBRfMaGkuOFKHntqYZmVk479FpOE8Yh3AQMZ1hfJEXSm1MBm0LjrAXPUBg86c4eOAJlaFUFXLae4CjfAxQtW/k/tp0VdFe+txap90CEOtONDuJ/Of7Yn/F6cIwtXe7qCovTkE5PrGUTRD3sGZmgIrmWnaSQeBUNUwu71eLY08aN8h+FQgWHhJfAHEfxjX0JJafl6cdIVSdb6xCqYKZ70cjrP1g1s15KuxAO86r4yZ1c+iXJXkf/6GJSg+0= ceph-53d92d24-c5b2-11ee-95b6-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-02-07T12:15:24.158 INFO:teuthology.orchestra.run.smithi184.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDBarTRTYp8QX+w0OGelEsd0ItvB1tEBc3Qrzn2Ec55HDhReXVvNORCT50/gca8iAHuH19ELRAUSMZo0t8qq5gdWa6paqtSoVuGX1GV770wZwoheHdRGldR9RP9ZBA87snuTDP+ESK4+LhU91gNt1vofZKiUTMWaEU2UwlQz3TbKQlzzk+XdvyqbiZNmtaa5kt6OdUQejBhDA9KeFyzfLjhLPpXRFLdyIqCbm6Gw9jPEY/3+5rlqcCV1BaQS/OVfNAz6AP+WmBRfMaGkuOFKHntqYZmVk479FpOE8Yh3AQMZ1hfJEXSm1MBm0LjrAXPUBg86c4eOAJlaFUFXLae4CjfAxQtW/k/tp0VdFe+txap90CEOtONDuJ/Of7Yn/F6cIwtXe7qCovTkE5PrGUTRD3sGZmgIrmWnaSQeBUNUwu71eLY08aN8h+FQgWHhJfAHEfxjX0JJafl6cdIVSdb6xCqYKZ70cjrP1g1s15KuxAO86r4yZ1c+iXJXkf/6GJSg+0= ceph-53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:15:24.173 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-02-07T12:15:24.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:24 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/1419393557' entity='client.admin' 2024-02-07T12:15:27.033 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:26 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:27.034 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:26 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd/host:smithi174", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:15:27.034 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:26 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:27.034 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:26 smithi174 ceph-mon[24580]: Deploying daemon alertmanager.smithi174 on smithi174 2024-02-07T12:15:27.879 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-02-07T12:15:27.880 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-02-07T12:15:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:28 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/267233567' entity='client.admin' 2024-02-07T12:15:30.972 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi184 2024-02-07T12:15:30.972 DEBUG:teuthology.orchestra.run.smithi184:> set -ex 2024-02-07T12:15:30.972 DEBUG:teuthology.orchestra.run.smithi184:> dd of=/etc/ceph/ceph.conf 2024-02-07T12:15:30.990 DEBUG:teuthology.orchestra.run.smithi184:> set -ex 2024-02-07T12:15:30.990 DEBUG:teuthology.orchestra.run.smithi184:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:15:31.048 INFO:tasks.cephadm:Adding host smithi184 to orchestrator... 2024-02-07T12:15:31.048 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch host add smithi184 2024-02-07T12:15:31.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:31 smithi174 ceph-mon[24580]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:31.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:31 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:34 smithi174 ceph-mon[24580]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:35.390 INFO:teuthology.orchestra.run.smithi174.stdout:Added host 'smithi184' with addr '172.21.15.184' 2024-02-07T12:15:36.245 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:35 smithi174 ceph-mon[24580]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi184", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:36.246 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:35 smithi174 ceph-mon[24580]: Deploying cephadm binary to smithi184 2024-02-07T12:15:36.246 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:35 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:36.770 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch host ls --format=json 2024-02-07T12:15:37.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:37 smithi174 ceph-mon[24580]: Added host smithi184 2024-02-07T12:15:37.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:37 smithi174 ceph-mon[24580]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:39.284 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:39 smithi174 ceph-mon[24580]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:39.788 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:15:39.789 INFO:teuthology.orchestra.run.smithi174.stdout:[{"addr": "172.21.15.174", "hostname": "smithi174", "labels": [], "status": ""}, {"addr": "172.21.15.184", "hostname": "smithi184", "labels": [], "status": ""}] 2024-02-07T12:15:40.934 INFO:tasks.cephadm:Setting crush tunables to default 2024-02-07T12:15:40.934 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd crush tunables default 2024-02-07T12:15:41.205 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:41 smithi174 ceph-mon[24580]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:15:41.206 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:41 smithi174 ceph-mon[24580]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:41.206 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:41 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:41.206 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:41 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:15:41.206 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:41 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-02-07T12:15:41.206 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:41 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:15:42.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:42 smithi174 ceph-mon[24580]: Deploying daemon crash.smithi174 on smithi174 2024-02-07T12:15:43.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:43 smithi174 ceph-mon[24580]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:45.014 INFO:teuthology.orchestra.run.smithi174.stderr:adjusted tunables profile to default 2024-02-07T12:15:45.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:44 smithi174 ceph-mon[24580]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:45.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:44 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:45.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:44 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/372505931' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-02-07T12:15:45.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:44 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:45.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:44 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:45.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:44 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:45.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:44 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:15:45.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:44 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:45.598 INFO:tasks.cephadm:Adding mon.smithi174 on smithi174 2024-02-07T12:15:45.599 INFO:tasks.cephadm:Adding mon.smithi184 on smithi184 2024-02-07T12:15:45.599 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch apply mon '2;smithi174:172.21.15.174=smithi174;smithi184:172.21.15.184=smithi184' 2024-02-07T12:15:46.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:46 smithi174 ceph-mon[24580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:15:46.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:46 smithi174 ceph-mon[24580]: Deploying daemon grafana.smithi174 on smithi174 2024-02-07T12:15:46.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:46 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/372505931' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-02-07T12:15:46.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:46 smithi174 ceph-mon[24580]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:15:47.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:47 smithi174 ceph-mon[24580]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:47.359 INFO:teuthology.orchestra.run.smithi184.stdout:Scheduled mon update... 2024-02-07T12:15:47.857 DEBUG:teuthology.orchestra.run.smithi184:mon.smithi184> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi184.service 2024-02-07T12:15:47.859 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:15:47.859 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:15:48.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:48 smithi174 ceph-mon[24580]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi174:172.21.15.174=smithi174;smithi184:172.21.15.184=smithi184", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:15:48.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:48 smithi174 ceph-mon[24580]: Saving service mon spec with placement smithi174:172.21.15.174=smithi174;smithi184:172.21.15.184=smithi184;count:2 2024-02-07T12:15:48.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:48 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:48.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:48 smithi174 ceph-mon[24580]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:49.576 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:15:49.577 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:15:49.578 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:15:50.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:49 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:15:50.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:49 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/52957327' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:15:51.142 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:15:51.142 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:15:51.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:50 smithi174 ceph-mon[24580]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:52.849 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:15:52.850 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:15:52.851 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:15:53.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:53 smithi174 ceph-mon[24580]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:53.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:53 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/3875780058' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:15:54.378 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:15:54.378 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:15:55.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:55 smithi174 ceph-mon[24580]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:56.095 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:15:56.095 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:15:56.096 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:15:57.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:57 smithi174 ceph-mon[24580]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:57.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:57 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/1576948963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:15:57.614 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:15:57.615 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:15:59.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:15:59 smithi174 ceph-mon[24580]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:15:59.320 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:15:59.321 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:15:59.322 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:00.294 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:00 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/1159214493' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:00.902 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:00.902 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:01.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:01 smithi174 ceph-mon[24580]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:01.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:01 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:02.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:02 smithi174 ceph-mon[24580]: Deploying daemon node-exporter.smithi174 on smithi174 2024-02-07T12:16:02.597 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:02.597 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:16:02.599 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:03.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:03 smithi174 ceph-mon[24580]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:03.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:03 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/2376998821' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:04.112 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:04.113 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:05.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:04 smithi174 ceph-mon[24580]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:05.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:04 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:05.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:04 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:05.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:04 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-02-07T12:16:05.844 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:05.844 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:16:05.846 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:06.446 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:06 smithi174 ceph-mon[24580]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-02-07T12:16:06.446 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:06 smithi174 ceph-mon[24580]: mgrmap e14: smithi174.whsozo(active, since 51s) 2024-02-07T12:16:06.446 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:06 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/3028271584' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:07.416 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:07.417 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:09.205 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:09.205 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:16:09.207 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:09.400 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:09 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/2957178947' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:10.736 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:10.737 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:12.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:12 smithi174 ceph-mon[24580]: Active manager daemon smithi174.whsozo restarted 2024-02-07T12:16:12.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:12 smithi174 ceph-mon[24580]: Activating manager daemon smithi174.whsozo 2024-02-07T12:16:12.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:12 smithi174 ceph-mon[24580]: osdmap e5: 0 total, 0 up, 0 in 2024-02-07T12:16:12.536 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:12.536 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:16:12.538 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:13.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: mgrmap e15: smithi174.whsozo(active, starting, since 0.968132s) 2024-02-07T12:16:13.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:16:13.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:16:13.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:16:13.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:16:13.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:16:13.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:16:13.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:16:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:16:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:16:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:16:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:16:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/415540673' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:14.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:14 smithi174 ceph-mon[24580]: mgrmap e16: smithi174.whsozo(active, since 1.97444s) 2024-02-07T12:16:14.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:14 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:14.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:14 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:14.144 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:14.144 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:15.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:15 smithi174 ceph-mon[24580]: [07/Feb/2024:12:16:13] ENGINE Bus STARTING 2024-02-07T12:16:15.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:15 smithi174 ceph-mon[24580]: [07/Feb/2024:12:16:13] ENGINE Serving on https://172.21.15.174:7150 2024-02-07T12:16:15.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:15 smithi174 ceph-mon[24580]: [07/Feb/2024:12:16:13] ENGINE Bus STARTED 2024-02-07T12:16:15.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:15 smithi174 ceph-mon[24580]: mgrmap e17: smithi174.whsozo(active, since 2s) 2024-02-07T12:16:15.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:15 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:16.339 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:16.339 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:16:16.341 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:16.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:16.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:16.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd/host:smithi174", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:16:16.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:16 smithi174 ceph-mon[24580]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:16:16.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:16 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/2227746181' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:18.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:17 smithi174 ceph-mon[24580]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:16:18.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:17 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:18.057 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:18.057 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:20.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:20.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:20.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd/host:smithi184", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:16:20.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:20.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:16:20.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-02-07T12:16:20.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:20.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:19 smithi174 ceph-mon[24580]: Deploying daemon crash.smithi184 on smithi184 2024-02-07T12:16:20.211 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:20.211 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:16:20.214 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:21.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:20 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/3416514474' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:21.945 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:21.945 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:23.760 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:23.760 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:16:23.761 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:24.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:23 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/611067225' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:25.260 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:25.261 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:25.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:25 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:25 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:16:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:25 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-02-07T12:16:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:25 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:16:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:25 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:25.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:25 smithi174 ceph-mon[24580]: Deploying daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:16:27.385 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:27.385 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":1,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:14:08.704203Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-02-07T12:16:27.386 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 1 2024-02-07T12:16:28.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:28 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/3129118804' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:28.990 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-02-07T12:16:28.990 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mon dump -f json 2024-02-07T12:16:29.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:29 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:29.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:29 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:16:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:29 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:29 smithi174 ceph-mon[24580]: Deploying daemon mon.smithi184 on smithi184 2024-02-07T12:16:30.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:30 smithi184 systemd[1]: Starting Ceph mon.smithi184 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:16:31.697 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:31 smithi184 podman[27061]: 2024-02-07 12:16:31.353783033 +0000 UTC m=+0.023469778 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-02-07T12:16:31.697 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:31 smithi184 podman[27061]: 2024-02-07 12:16:31.69506968 +0000 UTC m=+0.364756416 container create 11df1ddcd5336ed5ad94cf2a17f98d31e211fb4d5e91f05ec1a4d4231f6b97a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, release=754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, distribution-scope=public) 2024-02-07T12:16:32.377 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:31 smithi184 podman[27061]: 2024-02-07 12:16:31.973365098 +0000 UTC m=+0.643051834 container init 11df1ddcd5336ed5ad94cf2a17f98d31e211fb4d5e91f05ec1a4d4231f6b97a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vendor=Red Hat, Inc., io.buildah.version=1.19.8, release=754, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.component=centos-stream-container, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD) 2024-02-07T12:16:32.377 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:31 smithi184 podman[27061]: 2024-02-07 12:16:31.976412161 +0000 UTC m=+0.646098897 container start 11df1ddcd5336ed5ad94cf2a17f98d31e211fb4d5e91f05ec1a4d4231f6b97a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, version=8, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, name=centos-stream, io.buildah.version=1.19.8, GIT_CLEAN=True) 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:31 smithi184 ceph-mon[27098]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:31 smithi184 ceph-mon[27098]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:31 smithi184 ceph-mon[27098]: pidfile_write: ignore empty --pid-file 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: load: jerasure load: lrc 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: RocksDB version: 6.15.5 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Compile date Apr 18 2022 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: DB SUMMARY 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: DB Session ID: 4VZIRZOPHL11J9D7JRK6 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi184/store.db dir, Total Num: 0, files: 2024-02-07T12:16:32.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi184/store.db: 000004.log size: 511 ; 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.env: 0x55e409cc6860 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.fs: Posix File System 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.info_log: 0x55e40b2cffa0 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.statistics: (nil) 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.use_fsync: 0 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:16:32.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.db_log_dir: 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi184/store.db 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:16:32.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:16:32.382 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:16:32.382 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:16:32.382 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.write_buffer_manager: 0x55e40b3c02d0 2024-02-07T12:16:32.382 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.unordered_write: 0 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.row_cache: None 2024-02-07T12:16:32.383 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.wal_filter: None 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.preserve_deletes: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:16:32.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_open_files: -1 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:16:32.385 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Compression algorithms supported: 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: kZSTD supported: 0 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi184/store.db/MANIFEST-000003 2024-02-07T12:16:32.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.merge_operator: 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_filter: None 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e40b29bd00) 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: index_type: 0 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: data_block_index_type: 0 2024-02-07T12:16:32.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: index_shortening: 1 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: hash_index_allow_collision: 1 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: checksum: 1 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: no_block_cache: 0 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_cache: 0x55e40b306170 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_cache_options: 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: capacity : 536870912 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: num_shard_bits : 4 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: strict_capacity_limit : 0 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_cache_compressed: (nil) 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: persistent_cache: (nil) 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_size: 4096 2024-02-07T12:16:32.388 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_size_deviation: 10 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_restart_interval: 16 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: index_block_restart_interval: 1 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: metadata_block_size: 4096 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: partition_filters: 0 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: use_delta_encoding: 1 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: whole_key_filtering: 1 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: verify_compression: 0 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: format_version: 4 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: enable_index_compression: 1 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_align: 0 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compression: NoCompression 2024-02-07T12:16:32.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.num_levels: 7 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:16:32.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:16:32.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.arena_block_size: 4194304 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:16:32.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:16:32.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.ttl: 2592000 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi184/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308192002439, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-02-07T12:16:32.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308192002741, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1707308192, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "71eed56b-6432-4f3e-ad31-c55ca9a9dec9", "db_session_id": "4VZIRZOPHL11J9D7JRK6"}} 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308192003058, "job": 1, "event": "recovery_finished"} 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi184/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55e40b2eca80 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: DB pointer 0x55e40b2fc000 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** DB Stats ** 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:16:32.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** Compaction Stats [default] ** 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** Compaction Stats [default] ** 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:16:32.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** Compaction Stats [default] ** 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-02-07T12:16:32.397 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** Compaction Stats [default] ** 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:16:32.398 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184 does not exist in monmap, will attempt to join an existing cluster 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: using public_addr v2:172.21.15.184:0/0 -> [v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0] 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: starting mon.smithi184 rank -1 at public addrs [v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0] at bind addrs [v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi184 fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(???) e0 preinit fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).mds e1 new map 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).mds e1 print_map 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: e1 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: legacy client fscid: -1 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: No filesystems configured 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-02-07T12:16:32.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Deploying daemon crash.smithi174 on smithi174 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/372505931' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-02-07T12:16:32.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Deploying daemon grafana.smithi174 on smithi174 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/372505931' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: osdmap e4: 0 total, 0 up, 0 in 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi174:172.21.15.174=smithi174;smithi184:172.21.15.184=smithi184", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Saving service mon spec with placement smithi174:172.21.15.174=smithi174;smithi184:172.21.15.184=smithi184;count:2 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/52957327' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/3875780058' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/1576948963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/1159214493' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Deploying daemon node-exporter.smithi174 on smithi174 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/2376998821' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.402 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14160 172.21.15.174:0/3555579113' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mgrmap e14: smithi174.whsozo(active, since 51s) 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/3028271584' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/2957178947' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Active manager daemon smithi174.whsozo restarted 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Activating manager daemon smithi174.whsozo 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: osdmap e5: 0 total, 0 up, 0 in 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mgrmap e15: smithi174.whsozo(active, starting, since 0.968132s) 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:16:32.403 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/415540673' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mgrmap e16: smithi174.whsozo(active, since 1.97444s) 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: [07/Feb/2024:12:16:13] ENGINE Bus STARTING 2024-02-07T12:16:32.404 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: [07/Feb/2024:12:16:13] ENGINE Serving on https://172.21.15.174:7150 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: [07/Feb/2024:12:16:13] ENGINE Bus STARTED 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mgrmap e17: smithi174.whsozo(active, since 2s) 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd/host:smithi174", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/2227746181' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd/host:smithi184", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:16:32.405 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Deploying daemon crash.smithi184 on smithi184 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/3416514474' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/611067225' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Deploying daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/3129118804' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:32.406 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:32.407 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:16:32.407 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:32.407 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: Deploying daemon mon.smithi184 on smithi184 2024-02-07T12:16:32.407 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-02-07T12:16:32.407 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 ceph-mon[27098]: mon.smithi184@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-02-07T12:16:32.407 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 bash[27061]: 11df1ddcd5336ed5ad94cf2a17f98d31e211fb4d5e91f05ec1a4d4231f6b97a7 2024-02-07T12:16:32.407 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:32 smithi184 systemd[1]: Started Ceph mon.smithi184 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:16:37.078 INFO:teuthology.orchestra.run.smithi184.stdout: 2024-02-07T12:16:37.078 INFO:teuthology.orchestra.run.smithi184.stdout:{"epoch":2,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","modified":"2024-02-07T12:16:32.038696Z","created":"2024-02-07T12:14:08.704203Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi184","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:3300","nonce":0},{"type":"v1","addr":"172.21.15.184:6789","nonce":0}]},"addr":"172.21.15.184:6789/0","public_addr":"172.21.15.184:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-02-07T12:16:37.081 INFO:teuthology.orchestra.run.smithi184.stderr:dumped monmap epoch 2 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: mon.smithi174 calling monitor election 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/103092402' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: mon.smithi184 calling monitor election 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/1463499626' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: mon.smithi174 is new leader, mons smithi174,smithi184 in quorum (ranks 0,1) 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: monmap e2: 2 mons at {smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0],smithi184=[v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0]} 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: fsmap 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: osdmap e5: 0 total, 0 up, 0 in 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: mgrmap e17: smithi174.whsozo(active, since 26s) 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/1463499626' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: overall HEALTH_OK 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/1463499626' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/1463499626' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:37.380 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:16:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: mon.smithi174 calling monitor election 2024-02-07T12:16:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/103092402' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-02-07T12:16:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: mon.smithi184 calling monitor election 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/1463499626' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: mon.smithi174 is new leader, mons smithi174,smithi184 in quorum (ranks 0,1) 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: monmap e2: 2 mons at {smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0],smithi184=[v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0]} 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: fsmap 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: osdmap e5: 0 total, 0 up, 0 in 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: mgrmap e17: smithi174.whsozo(active, since 26s) 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:16:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/1463499626' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:16:37.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: overall HEALTH_OK 2024-02-07T12:16:37.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/1463499626' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:16:37.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/1463499626' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:16:37.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:37.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:37.693 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-02-07T12:16:37.693 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph config generate-minimal-conf 2024-02-07T12:16:38.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:38 smithi184 ceph-mon[27098]: Deploying daemon node-exporter.smithi184 on smithi184 2024-02-07T12:16:38.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:38 smithi184 ceph-mon[27098]: mgrmap e18: smithi174.whsozo(active, since 26s), standbys: smithi184.prgadw 2024-02-07T12:16:38.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:38 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi184.prgadw", "id": "smithi184.prgadw"}]: dispatch 2024-02-07T12:16:38.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:38 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:38.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:38 smithi174 ceph-mon[24580]: Deploying daemon node-exporter.smithi184 on smithi184 2024-02-07T12:16:38.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:38 smithi174 ceph-mon[24580]: mgrmap e18: smithi174.whsozo(active, since 26s), standbys: smithi184.prgadw 2024-02-07T12:16:38.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:38 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi184.prgadw", "id": "smithi184.prgadw"}]: dispatch 2024-02-07T12:16:38.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:38 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:16:39.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:39 smithi174 ceph-mon[24580]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:39 smithi184 ceph-mon[27098]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:39.578 INFO:teuthology.orchestra.run.smithi174.stdout:# minimal ceph.conf for 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:16:39.578 INFO:teuthology.orchestra.run.smithi174.stdout:[global] 2024-02-07T12:16:39.578 INFO:teuthology.orchestra.run.smithi174.stdout: fsid = 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:16:39.578 INFO:teuthology.orchestra.run.smithi174.stdout: mon_host = [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] [v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0] 2024-02-07T12:16:40.173 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-02-07T12:16:40.173 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:16:40.173 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/ceph/ceph.conf 2024-02-07T12:16:40.202 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:16:40.202 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:16:40.271 DEBUG:teuthology.orchestra.run.smithi184:> set -ex 2024-02-07T12:16:40.271 DEBUG:teuthology.orchestra.run.smithi184:> sudo dd of=/etc/ceph/ceph.conf 2024-02-07T12:16:40.298 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:40 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3845879497' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:40.303 DEBUG:teuthology.orchestra.run.smithi184:> set -ex 2024-02-07T12:16:40.303 DEBUG:teuthology.orchestra.run.smithi184:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:16:40.370 INFO:tasks.cephadm:Deploying OSDs... 2024-02-07T12:16:40.370 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:16:40.370 DEBUG:teuthology.orchestra.run.smithi174:> dd if=/scratch_devs of=/dev/stdout 2024-02-07T12:16:40.386 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-02-07T12:16:40.386 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_1 2024-02-07T12:16:40.441 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-02-07T12:16:40.441 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:16:40.441 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 644 Links: 1 2024-02-07T12:16:40.441 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:16:40.441 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:16:40.441 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-02-07 12:16:01.295944853 +0000 2024-02-07T12:16:40.442 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-02-07 12:13:16.099833417 +0000 2024-02-07T12:16:40.442 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-02-07 12:13:16.099833417 +0000 2024-02-07T12:16:40.442 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-02-07 12:13:16.099833417 +0000 2024-02-07T12:16:40.442 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-02-07T12:16:40.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:40 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3845879497' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:40.506 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-02-07T12:16:40.507 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-02-07T12:16:40.507 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000213107 s, 2.4 MB/s 2024-02-07T12:16:40.508 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-02-07T12:16:40.564 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_2 2024-02-07T12:16:40.619 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-02-07T12:16:40.619 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:16:40.619 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 663 Links: 1 2024-02-07T12:16:40.619 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:16:40.619 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:16:40.619 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-02-07 12:16:01.296944840 +0000 2024-02-07T12:16:40.620 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-02-07 12:13:16.443828968 +0000 2024-02-07T12:16:40.620 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-02-07 12:13:16.443828968 +0000 2024-02-07T12:16:40.620 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-02-07 12:13:16.443828968 +0000 2024-02-07T12:16:40.620 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-02-07T12:16:40.685 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-02-07T12:16:40.685 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-02-07T12:16:40.686 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000188909 s, 2.7 MB/s 2024-02-07T12:16:40.686 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-02-07T12:16:40.742 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_3 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 679 Links: 1 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-02-07 12:16:01.296944840 +0000 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-02-07 12:13:16.786824532 +0000 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-02-07 12:13:16.786824532 +0000 2024-02-07T12:16:40.797 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-02-07 12:13:16.786824532 +0000 2024-02-07T12:16:40.798 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-02-07T12:16:40.863 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-02-07T12:16:40.863 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-02-07T12:16:40.863 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000183964 s, 2.8 MB/s 2024-02-07T12:16:40.864 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-02-07T12:16:40.920 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_4 2024-02-07T12:16:40.975 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-02-07T12:16:40.975 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:16:40.976 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 700 Links: 1 2024-02-07T12:16:40.976 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:16:40.976 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:16:40.976 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-02-07 12:16:01.296944840 +0000 2024-02-07T12:16:40.976 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-02-07 12:13:17.307817794 +0000 2024-02-07T12:16:40.976 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-02-07 12:13:17.307817794 +0000 2024-02-07T12:16:40.976 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-02-07 12:13:17.307817794 +0000 2024-02-07T12:16:40.976 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-02-07T12:16:41.041 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-02-07T12:16:41.041 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-02-07T12:16:41.041 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000191119 s, 2.7 MB/s 2024-02-07T12:16:41.043 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-02-07T12:16:41.098 DEBUG:teuthology.orchestra.run.smithi184:> set -ex 2024-02-07T12:16:41.099 DEBUG:teuthology.orchestra.run.smithi184:> dd if=/scratch_devs of=/dev/stdout 2024-02-07T12:16:41.114 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-02-07T12:16:41.115 DEBUG:teuthology.orchestra.run.smithi184:> stat /dev/vg_nvme/lv_1 2024-02-07T12:16:41.169 INFO:teuthology.orchestra.run.smithi184.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-02-07T12:16:41.169 INFO:teuthology.orchestra.run.smithi184.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:16:41.169 INFO:teuthology.orchestra.run.smithi184.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-02-07T12:16:41.169 INFO:teuthology.orchestra.run.smithi184.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:16:41.169 INFO:teuthology.orchestra.run.smithi184.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:16:41.170 INFO:teuthology.orchestra.run.smithi184.stdout:Access: 2024-02-07 12:16:38.066336936 +0000 2024-02-07T12:16:41.170 INFO:teuthology.orchestra.run.smithi184.stdout:Modify: 2024-02-07 12:13:16.105954856 +0000 2024-02-07T12:16:41.170 INFO:teuthology.orchestra.run.smithi184.stdout:Change: 2024-02-07 12:13:16.105954856 +0000 2024-02-07T12:16:41.170 INFO:teuthology.orchestra.run.smithi184.stdout: Birth: 2024-02-07 12:13:16.105954856 +0000 2024-02-07T12:16:41.170 DEBUG:teuthology.orchestra.run.smithi184:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-02-07T12:16:41.236 INFO:teuthology.orchestra.run.smithi184.stderr:1+0 records in 2024-02-07T12:16:41.236 INFO:teuthology.orchestra.run.smithi184.stderr:1+0 records out 2024-02-07T12:16:41.236 INFO:teuthology.orchestra.run.smithi184.stderr:512 bytes copied, 0.000271331 s, 1.9 MB/s 2024-02-07T12:16:41.237 DEBUG:teuthology.orchestra.run.smithi184:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-02-07T12:16:41.293 DEBUG:teuthology.orchestra.run.smithi184:> stat /dev/vg_nvme/lv_2 2024-02-07T12:16:41.349 INFO:teuthology.orchestra.run.smithi184.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-02-07T12:16:41.349 INFO:teuthology.orchestra.run.smithi184.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:16:41.349 INFO:teuthology.orchestra.run.smithi184.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-02-07T12:16:41.349 INFO:teuthology.orchestra.run.smithi184.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:16:41.349 INFO:teuthology.orchestra.run.smithi184.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:16:41.349 INFO:teuthology.orchestra.run.smithi184.stdout:Access: 2024-02-07 12:16:38.066336936 +0000 2024-02-07T12:16:41.350 INFO:teuthology.orchestra.run.smithi184.stdout:Modify: 2024-02-07 12:13:16.458947072 +0000 2024-02-07T12:16:41.350 INFO:teuthology.orchestra.run.smithi184.stdout:Change: 2024-02-07 12:13:16.458947072 +0000 2024-02-07T12:16:41.350 INFO:teuthology.orchestra.run.smithi184.stdout: Birth: 2024-02-07 12:13:16.458947072 +0000 2024-02-07T12:16:41.350 DEBUG:teuthology.orchestra.run.smithi184:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-02-07T12:16:41.412 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:41 smithi184 ceph-mon[27098]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:41.412 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:41 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:41.414 INFO:teuthology.orchestra.run.smithi184.stderr:1+0 records in 2024-02-07T12:16:41.415 INFO:teuthology.orchestra.run.smithi184.stderr:1+0 records out 2024-02-07T12:16:41.415 INFO:teuthology.orchestra.run.smithi184.stderr:512 bytes copied, 0.00019559 s, 2.6 MB/s 2024-02-07T12:16:41.416 DEBUG:teuthology.orchestra.run.smithi184:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-02-07T12:16:41.472 DEBUG:teuthology.orchestra.run.smithi184:> stat /dev/vg_nvme/lv_3 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout:Access: 2024-02-07 12:16:38.066336936 +0000 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout:Modify: 2024-02-07 12:13:16.815939198 +0000 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout:Change: 2024-02-07 12:13:16.815939198 +0000 2024-02-07T12:16:41.528 INFO:teuthology.orchestra.run.smithi184.stdout: Birth: 2024-02-07 12:13:16.815939198 +0000 2024-02-07T12:16:41.529 DEBUG:teuthology.orchestra.run.smithi184:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-02-07T12:16:41.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:41 smithi174 ceph-mon[24580]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:41.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:41 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:41.593 INFO:teuthology.orchestra.run.smithi184.stderr:1+0 records in 2024-02-07T12:16:41.593 INFO:teuthology.orchestra.run.smithi184.stderr:1+0 records out 2024-02-07T12:16:41.593 INFO:teuthology.orchestra.run.smithi184.stderr:512 bytes copied, 0.000211193 s, 2.4 MB/s 2024-02-07T12:16:41.594 DEBUG:teuthology.orchestra.run.smithi184:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-02-07T12:16:41.650 DEBUG:teuthology.orchestra.run.smithi184:> stat /dev/vg_nvme/lv_4 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout:Context: system_u:object_r:device_t:s0 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout:Access: 2024-02-07 12:16:38.067336914 +0000 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout:Modify: 2024-02-07 12:13:17.160931590 +0000 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout:Change: 2024-02-07 12:13:17.160931590 +0000 2024-02-07T12:16:41.706 INFO:teuthology.orchestra.run.smithi184.stdout: Birth: 2024-02-07 12:13:17.160931590 +0000 2024-02-07T12:16:41.707 DEBUG:teuthology.orchestra.run.smithi184:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-02-07T12:16:41.772 INFO:teuthology.orchestra.run.smithi184.stderr:1+0 records in 2024-02-07T12:16:41.773 INFO:teuthology.orchestra.run.smithi184.stderr:1+0 records out 2024-02-07T12:16:41.773 INFO:teuthology.orchestra.run.smithi184.stderr:512 bytes copied, 0.000214816 s, 2.4 MB/s 2024-02-07T12:16:41.774 DEBUG:teuthology.orchestra.run.smithi184:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-02-07T12:16:41.830 INFO:tasks.cephadm:Deploying osd.0 on smithi174 with /dev/vg_nvme/lv_4... 2024-02-07T12:16:41.830 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-02-07T12:16:42.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:42 smithi184 ceph-mon[27098]: Deploying daemon prometheus.smithi174 on smithi174 2024-02-07T12:16:42.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:42 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:42.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:42 smithi174 ceph-mon[24580]: Deploying daemon prometheus.smithi174 on smithi174 2024-02-07T12:16:42.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:42 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:43 smithi184 ceph-mon[27098]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:43 smithi174 ceph-mon[24580]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:44.599 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-02-07T12:16:44.599 INFO:teuthology.orchestra.run.smithi174.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-02-07T12:16:44.599 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10+0 records in 2024-02-07T12:16:44.599 INFO:teuthology.orchestra.run.smithi174.stderr:10+0 records out 2024-02-07T12:16:44.599 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0117878 s, 890 MB/s 2024-02-07T12:16:44.599 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping successful for: 2024-02-07T12:16:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:45 smithi184 ceph-mon[27098]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:45.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:45 smithi174 ceph-mon[24580]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:45.596 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi174:vg_nvme/lv_4 2024-02-07T12:16:47.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:47 smithi174 ceph-mon[24580]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:47 smithi184 ceph-mon[27098]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:49.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:49 smithi174 ceph-mon[24580]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:49.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:49 smithi184 ceph-mon[27098]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:50.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:50 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:16:50.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:50 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:16:50.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:50 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:50.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:50 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:16:50.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:50 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:16:50.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:50 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:51.279 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:51 smithi174 ceph-mon[24580]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:51.279 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:51 smithi174 ceph-mon[24580]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:16:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:51 smithi184 ceph-mon[27098]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:51 smithi184 ceph-mon[27098]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:16:53.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:53 smithi174 ceph-mon[24580]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:53 smithi184 ceph-mon[27098]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:55.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:55 smithi174 ceph-mon[24580]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:55.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:55 smithi184 ceph-mon[27098]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:57.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:57.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:16:57.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:57.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:16:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:56 smithi174 ceph-mon[24580]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:57.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:57.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:16:57.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:16:57.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:16:57.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:56 smithi184 ceph-mon[27098]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:58.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:57 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2431207302' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67efbac6-68ee-4b1d-b17b-30ccdecd5fd4"}]: dispatch 2024-02-07T12:16:58.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:57 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2431207302' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67efbac6-68ee-4b1d-b17b-30ccdecd5fd4"}]': finished 2024-02-07T12:16:58.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:57 smithi184 ceph-mon[27098]: osdmap e6: 1 total, 0 up, 1 in 2024-02-07T12:16:58.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:16:58.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:58.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:58.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:57 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3309810629' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:16:58.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:57 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2431207302' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67efbac6-68ee-4b1d-b17b-30ccdecd5fd4"}]: dispatch 2024-02-07T12:16:58.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:57 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2431207302' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67efbac6-68ee-4b1d-b17b-30ccdecd5fd4"}]': finished 2024-02-07T12:16:58.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:57 smithi174 ceph-mon[24580]: osdmap e6: 1 total, 0 up, 1 in 2024-02-07T12:16:58.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:16:58.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:58.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:58.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:57 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3309810629' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:16:59.343 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:59 smithi174 ceph-mon[24580]: Updating smithi184:/etc/ceph/ceph.conf 2024-02-07T12:16:59.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:59 smithi174 ceph-mon[24580]: Updating smithi184:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:16:59.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:59 smithi174 ceph-mon[24580]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:59.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:16:59 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:16:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:59 smithi184 ceph-mon[27098]: Updating smithi184:/etc/ceph/ceph.conf 2024-02-07T12:16:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:59 smithi184 ceph-mon[27098]: Updating smithi184:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:16:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:59 smithi184 ceph-mon[27098]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:16:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:16:59 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:01 smithi174 ceph-mon[24580]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:17:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:17:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:01 smithi184 ceph-mon[27098]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:01.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:17:01.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:17:01.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:02.157 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:02 smithi174 ceph-mon[24580]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:17:02.158 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:02 smithi174 ceph-mon[24580]: Reconfiguring mon.smithi174 (unknown last config time)... 2024-02-07T12:17:02.158 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:02 smithi174 ceph-mon[24580]: Reconfiguring daemon mon.smithi174 on smithi174 2024-02-07T12:17:02.158 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:02 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:02.158 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:02 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.whsozo", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:17:02.158 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:02 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:17:02.158 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:02 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:02 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:17:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:02 smithi184 ceph-mon[27098]: Reconfiguring mon.smithi174 (unknown last config time)... 2024-02-07T12:17:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:02 smithi184 ceph-mon[27098]: Reconfiguring daemon mon.smithi174 on smithi174 2024-02-07T12:17:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:02 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:02 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.whsozo", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:17:02.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:02 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:17:02.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:02 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:03.372 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:03 smithi174 ceph-mon[24580]: Reconfiguring mgr.smithi174.whsozo (unknown last config time)... 2024-02-07T12:17:03.373 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:03 smithi174 ceph-mon[24580]: Reconfiguring daemon mgr.smithi174.whsozo on smithi174 2024-02-07T12:17:03.373 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:03 smithi174 ceph-mon[24580]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:03 smithi184 ceph-mon[27098]: Reconfiguring mgr.smithi174.whsozo (unknown last config time)... 2024-02-07T12:17:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:03 smithi184 ceph-mon[27098]: Reconfiguring daemon mgr.smithi174.whsozo on smithi174 2024-02-07T12:17:03.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:03 smithi184 ceph-mon[27098]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:05.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:05 smithi184 ceph-mon[27098]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:05.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:05 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:05.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:05 smithi184 ceph-mon[27098]: Reconfiguring alertmanager.smithi174 (dependencies changed)... 2024-02-07T12:17:05.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:05 smithi184 ceph-mon[27098]: Reconfiguring daemon alertmanager.smithi174 on smithi174 2024-02-07T12:17:05.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:05 smithi174 ceph-mon[24580]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:05.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:05 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:05.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:05 smithi174 ceph-mon[24580]: Reconfiguring alertmanager.smithi174 (dependencies changed)... 2024-02-07T12:17:05.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:05 smithi174 ceph-mon[24580]: Reconfiguring daemon alertmanager.smithi174 on smithi174 2024-02-07T12:17:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:06 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:17:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:06 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:06.531 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:06 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:17:06.532 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:06 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:07.411 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:07 smithi174 ceph-mon[24580]: Deploying daemon osd.0 on smithi174 2024-02-07T12:17:07.411 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:07 smithi174 ceph-mon[24580]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:07 smithi184 ceph-mon[27098]: Deploying daemon osd.0 on smithi174 2024-02-07T12:17:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:07 smithi184 ceph-mon[27098]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:09.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:09 smithi184 ceph-mon[27098]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:09.524 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:09 smithi174 ceph-mon[24580]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:11.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:11 smithi174 ceph-mon[24580]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:11.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:11.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:11 smithi174 ceph-mon[24580]: Reconfiguring crash.smithi174 (monmap changed)... 2024-02-07T12:17:11.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:17:11.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:11.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:11 smithi174 ceph-mon[24580]: Reconfiguring daemon crash.smithi174 on smithi174 2024-02-07T12:17:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:11 smithi184 ceph-mon[27098]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:11 smithi184 ceph-mon[27098]: Reconfiguring crash.smithi174 (monmap changed)... 2024-02-07T12:17:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:17:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:11.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:11 smithi184 ceph-mon[27098]: Reconfiguring daemon crash.smithi174 on smithi174 2024-02-07T12:17:13.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:13 smithi174 ceph-mon[24580]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:13.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:17:13.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:17:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:13 smithi184 ceph-mon[27098]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:17:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:17:15.114 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:15 smithi174 ceph-mon[24580]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:15.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:15 smithi184 ceph-mon[27098]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:16.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:16.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:16 smithi174 ceph-mon[24580]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:16.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:16 smithi184 ceph-mon[27098]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:18.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:18 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:18.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:18 smithi174 ceph-mon[24580]: Reconfiguring grafana.smithi174 (dependencies changed)... 2024-02-07T12:17:18.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:18 smithi174 ceph-mon[24580]: Reconfiguring daemon grafana.smithi174 on smithi174 2024-02-07T12:17:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:18 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:18 smithi184 ceph-mon[27098]: Reconfiguring grafana.smithi174 (dependencies changed)... 2024-02-07T12:17:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:18 smithi184 ceph-mon[27098]: Reconfiguring daemon grafana.smithi174 on smithi174 2024-02-07T12:17:18.916 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 0 on host 'smithi174' 2024-02-07T12:17:19.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:19 smithi174 ceph-mon[24580]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:19.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:19 smithi174 ceph-mon[24580]: from='osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:17:19.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:19.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:19 smithi184 ceph-mon[27098]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:19.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:19 smithi184 ceph-mon[27098]: from='osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:17:19.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:19.668 DEBUG:teuthology.orchestra.run.smithi174:osd.0> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.0.service 2024-02-07T12:17:19.671 INFO:tasks.cephadm:Deploying osd.1 on smithi174 with /dev/vg_nvme/lv_3... 2024-02-07T12:17:19.671 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-02-07T12:17:20.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:20 smithi174 ceph-mon[24580]: from='osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:17:20.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:20 smithi174 ceph-mon[24580]: osdmap e7: 1 total, 0 up, 1 in 2024-02-07T12:17:20.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:20 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:17:20.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:20 smithi174 ceph-mon[24580]: from='osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:17:20.297 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:17:20 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[34985]: 2024-02-07T12:17:20.014+0000 7f1608afa700 -1 osd.0 0 waiting for initial osdmap 2024-02-07T12:17:20.297 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:17:20 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[34985]: 2024-02-07T12:17:20.017+0000 7f1603491700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:17:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:20 smithi184 ceph-mon[27098]: from='osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:17:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:20 smithi184 ceph-mon[27098]: osdmap e7: 1 total, 0 up, 1 in 2024-02-07T12:17:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:20 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:17:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:20 smithi184 ceph-mon[27098]: from='osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:17:21.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:21 smithi174 ceph-mon[24580]: from='osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-02-07T12:17:21.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:21 smithi174 ceph-mon[24580]: osdmap e8: 1 total, 0 up, 1 in 2024-02-07T12:17:21.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:21 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:17:21.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:21 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:17:21.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:21 smithi174 ceph-mon[24580]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:21 smithi184 ceph-mon[27098]: from='osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-02-07T12:17:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:21 smithi184 ceph-mon[27098]: osdmap e8: 1 total, 0 up, 1 in 2024-02-07T12:17:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:21 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:17:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:21 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:17:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:21 smithi184 ceph-mon[27098]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-02-07T12:17:22.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:22 smithi174 ceph-mon[24580]: purged_snaps scrub starts 2024-02-07T12:17:22.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:22 smithi174 ceph-mon[24580]: purged_snaps scrub ok 2024-02-07T12:17:22.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:22 smithi174 ceph-mon[24580]: osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960] boot 2024-02-07T12:17:22.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:22 smithi174 ceph-mon[24580]: osdmap e9: 1 total, 1 up, 1 in 2024-02-07T12:17:22.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:17:22.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:22 smithi184 ceph-mon[27098]: purged_snaps scrub starts 2024-02-07T12:17:22.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:22 smithi184 ceph-mon[27098]: purged_snaps scrub ok 2024-02-07T12:17:22.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:22 smithi184 ceph-mon[27098]: osd.0 [v2:172.21.15.174:6802/3253424960,v1:172.21.15.174:6803/3253424960] boot 2024-02-07T12:17:22.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:22 smithi184 ceph-mon[27098]: osdmap e9: 1 total, 1 up, 1 in 2024-02-07T12:17:22.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:17:23.132 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:23 smithi174 ceph-mon[24580]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:23.132 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:23 smithi174 ceph-mon[24580]: osdmap e10: 1 total, 1 up, 1 in 2024-02-07T12:17:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:23 smithi184 ceph-mon[27098]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:23 smithi184 ceph-mon[27098]: osdmap e10: 1 total, 1 up, 1 in 2024-02-07T12:17:23.607 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-02-07T12:17:23.608 INFO:teuthology.orchestra.run.smithi174.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-02-07T12:17:23.608 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10+0 records in 2024-02-07T12:17:23.608 INFO:teuthology.orchestra.run.smithi174.stderr:10+0 records out 2024-02-07T12:17:23.608 INFO:teuthology.orchestra.run.smithi174.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0114229 s, 918 MB/s 2024-02-07T12:17:23.608 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 2024-02-07T12:17:23.608 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping successful for: 2024-02-07T12:17:24.774 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi174:vg_nvme/lv_3 2024-02-07T12:17:25.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:25 smithi174 ceph-mon[24580]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:25.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:25 smithi184 ceph-mon[27098]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:26.435 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:26 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:26.435 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:26 smithi174 ceph-mon[24580]: Reconfiguring crash.smithi184 (monmap changed)... 2024-02-07T12:17:26.435 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:26 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:17:26.435 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:26 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:26.435 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:26 smithi174 ceph-mon[24580]: Reconfiguring daemon crash.smithi184 on smithi184 2024-02-07T12:17:26.436 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:26 smithi174 ceph-mon[24580]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:26.635 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:26 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:26.635 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:26 smithi184 ceph-mon[27098]: Reconfiguring crash.smithi184 (monmap changed)... 2024-02-07T12:17:26.635 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:26 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:17:26.635 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:26 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:26.635 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:26 smithi184 ceph-mon[27098]: Reconfiguring daemon crash.smithi184 on smithi184 2024-02-07T12:17:26.635 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:26 smithi184 ceph-mon[27098]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: Reconfiguring mgr.smithi184.prgadw (monmap changed)... 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: Reconfiguring daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:17:27.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:27.936 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:27.936 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: Reconfiguring mgr.smithi184.prgadw (monmap changed)... 2024-02-07T12:17:27.936 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:17:27.936 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:17:27.936 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:27.936 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: Reconfiguring daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:17:27.937 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:17:27.937 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:17:27.937 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:17:27.937 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:29.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:29.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:28 smithi174 ceph-mon[24580]: Reconfiguring mon.smithi184 (monmap changed)... 2024-02-07T12:17:29.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:17:29.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:17:29.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:29.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:28 smithi174 ceph-mon[24580]: Reconfiguring daemon mon.smithi184 on smithi184 2024-02-07T12:17:29.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:28 smithi174 ceph-mon[24580]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:28 smithi184 ceph-mon[27098]: Reconfiguring mon.smithi184 (monmap changed)... 2024-02-07T12:17:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:17:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:17:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:28 smithi184 ceph-mon[27098]: Reconfiguring daemon mon.smithi184 on smithi184 2024-02-07T12:17:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:28 smithi184 ceph-mon[27098]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:30.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:29 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:30.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:29 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:17:30.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:29 smithi174 ceph-mon[24580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:17:30.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:29 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/394894932' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "648a2717-71c4-4f4b-a47b-1ef7562ce576"}]: dispatch 2024-02-07T12:17:30.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:29 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/394894932' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "648a2717-71c4-4f4b-a47b-1ef7562ce576"}]': finished 2024-02-07T12:17:30.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:29 smithi174 ceph-mon[24580]: osdmap e11: 2 total, 1 up, 2 in 2024-02-07T12:17:30.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:29 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:29 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:29 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:17:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:29 smithi184 ceph-mon[27098]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:17:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:29 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/394894932' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "648a2717-71c4-4f4b-a47b-1ef7562ce576"}]: dispatch 2024-02-07T12:17:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:29 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/394894932' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "648a2717-71c4-4f4b-a47b-1ef7562ce576"}]': finished 2024-02-07T12:17:30.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:29 smithi184 ceph-mon[27098]: osdmap e11: 2 total, 1 up, 2 in 2024-02-07T12:17:30.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:29 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:31.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:30 smithi174 ceph-mon[24580]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:31.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:30 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3182510248' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:17:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:30 smithi184 ceph-mon[27098]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:30 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3182510248' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:17:33.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:33 smithi174 ceph-mon[24580]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:33 smithi184 ceph-mon[27098]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:35.206 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:35 smithi174 ceph-mon[24580]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:35 smithi184 ceph-mon[27098]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:36.293 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:36 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:17:36.293 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:36 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:36 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:17:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:36 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:37.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:37 smithi174 ceph-mon[24580]: Deploying daemon osd.1 on smithi174 2024-02-07T12:17:37.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:37 smithi174 ceph-mon[24580]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:37 smithi184 ceph-mon[27098]: Deploying daemon osd.1 on smithi174 2024-02-07T12:17:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:37 smithi184 ceph-mon[27098]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:39.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:39 smithi184 ceph-mon[27098]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:39.460 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:39 smithi174 ceph-mon[24580]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:41.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:41 smithi174 ceph-mon[24580]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:41.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:41 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:41 smithi184 ceph-mon[27098]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:41 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:43 smithi184 ceph-mon[27098]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:43.467 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:43 smithi174 ceph-mon[24580]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:43.499 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 1 on host 'smithi174' 2024-02-07T12:17:44.160 DEBUG:teuthology.orchestra.run.smithi174:osd.1> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.1.service 2024-02-07T12:17:44.163 INFO:tasks.cephadm:Deploying osd.2 on smithi174 with /dev/vg_nvme/lv_2... 2024-02-07T12:17:44.163 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.174:9093"}]: dispatch 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.174:9093"}]: dispatch 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.174:3000"}]: dispatch 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.174:3000"}]: dispatch 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:17:44.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:17:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.174:9095"}]: dispatch 2024-02-07T12:17:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.174:9095"}]: dispatch 2024-02-07T12:17:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:17:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:17:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: from='osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:17:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:44 smithi174 ceph-mon[24580]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.174:9093"}]: dispatch 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.174:9093"}]: dispatch 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.174:3000"}]: dispatch 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.174:3000"}]: dispatch 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:17:44.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.174:9095"}]: dispatch 2024-02-07T12:17:44.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.174:9095"}]: dispatch 2024-02-07T12:17:44.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:44.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:17:44.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:44.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:17:44.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: from='osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:17:44.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:44 smithi184 ceph-mon[27098]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:45.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:45 smithi174 ceph-mon[24580]: from='osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:17:45.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:45 smithi174 ceph-mon[24580]: osdmap e12: 2 total, 1 up, 2 in 2024-02-07T12:17:45.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:45 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:45.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:45 smithi174 ceph-mon[24580]: from='osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:17:45.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:45 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:45.797 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:17:45 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[37853]: 2024-02-07T12:17:45.556+0000 7f4429bdc700 -1 osd.1 0 waiting for initial osdmap 2024-02-07T12:17:45.798 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:17:45 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[37853]: 2024-02-07T12:17:45.559+0000 7f4423571700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:17:45.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:45 smithi184 ceph-mon[27098]: from='osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:17:45.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:45 smithi184 ceph-mon[27098]: osdmap e12: 2 total, 1 up, 2 in 2024-02-07T12:17:45.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:45 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:45.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:45 smithi184 ceph-mon[27098]: from='osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:17:45.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:45 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:46.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:46 smithi174 ceph-mon[24580]: from='osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-02-07T12:17:46.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:46 smithi174 ceph-mon[24580]: osdmap e13: 2 total, 1 up, 2 in 2024-02-07T12:17:46.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:46.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:46.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:46 smithi174 ceph-mon[24580]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:46.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:46 smithi184 ceph-mon[27098]: from='osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-02-07T12:17:46.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:46 smithi184 ceph-mon[27098]: osdmap e13: 2 total, 1 up, 2 in 2024-02-07T12:17:46.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:46.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:46.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:46 smithi184 ceph-mon[27098]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-02-07T12:17:46.949 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-02-07T12:17:46.949 INFO:teuthology.orchestra.run.smithi174.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-02-07T12:17:46.949 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10+0 records in 2024-02-07T12:17:46.949 INFO:teuthology.orchestra.run.smithi174.stderr:10+0 records out 2024-02-07T12:17:46.949 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012627 s, 830 MB/s 2024-02-07T12:17:46.949 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping successful for: 2024-02-07T12:17:47.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:47 smithi174 ceph-mon[24580]: purged_snaps scrub starts 2024-02-07T12:17:47.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:47 smithi174 ceph-mon[24580]: purged_snaps scrub ok 2024-02-07T12:17:47.691 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:47 smithi174 ceph-mon[24580]: osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833] boot 2024-02-07T12:17:47.691 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:47 smithi174 ceph-mon[24580]: osdmap e14: 2 total, 2 up, 2 in 2024-02-07T12:17:47.691 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:47 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:47.716 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi174:vg_nvme/lv_2 2024-02-07T12:17:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:47 smithi184 ceph-mon[27098]: purged_snaps scrub starts 2024-02-07T12:17:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:47 smithi184 ceph-mon[27098]: purged_snaps scrub ok 2024-02-07T12:17:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:47 smithi184 ceph-mon[27098]: osd.1 [v2:172.21.15.174:6810/3157081833,v1:172.21.15.174:6811/3157081833] boot 2024-02-07T12:17:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:47 smithi184 ceph-mon[27098]: osdmap e14: 2 total, 2 up, 2 in 2024-02-07T12:17:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:47 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:17:48.731 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:48 smithi174 ceph-mon[24580]: osdmap e15: 2 total, 2 up, 2 in 2024-02-07T12:17:48.731 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:48 smithi174 ceph-mon[24580]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:48.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:48 smithi184 ceph-mon[27098]: osdmap e15: 2 total, 2 up, 2 in 2024-02-07T12:17:48.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:48 smithi184 ceph-mon[27098]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:50.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:17:50.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:17:50.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:50.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:17:50.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:17:50.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:17:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:50 smithi174 ceph-mon[24580]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:17:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:50 smithi174 ceph-mon[24580]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:50 smithi184 ceph-mon[27098]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:17:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:50 smithi184 ceph-mon[27098]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:53.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:53 smithi174 ceph-mon[24580]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:53.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:53 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:53 smithi184 ceph-mon[27098]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:53 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:17:54.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:54 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3598654284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c005d20a-a54c-4d99-bf9b-18c11f2accf5"}]: dispatch 2024-02-07T12:17:54.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:54 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3598654284' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c005d20a-a54c-4d99-bf9b-18c11f2accf5"}]': finished 2024-02-07T12:17:54.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:54 smithi174 ceph-mon[24580]: osdmap e16: 3 total, 2 up, 3 in 2024-02-07T12:17:54.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:54 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:17:54.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:54 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3598654284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c005d20a-a54c-4d99-bf9b-18c11f2accf5"}]: dispatch 2024-02-07T12:17:54.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:54 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3598654284' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c005d20a-a54c-4d99-bf9b-18c11f2accf5"}]': finished 2024-02-07T12:17:54.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:54 smithi184 ceph-mon[27098]: osdmap e16: 3 total, 2 up, 3 in 2024-02-07T12:17:54.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:54 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:17:55.349 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:55 smithi184 ceph-mon[27098]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:55.379 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:55 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/1185880376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:17:55.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:55 smithi174 ceph-mon[24580]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:55.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:55 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/1185880376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:17:57.290 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:57 smithi174 ceph-mon[24580]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:57 smithi184 ceph-mon[27098]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:59.135 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:17:59 smithi174 ceph-mon[24580]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:17:59.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:17:59 smithi184 ceph-mon[27098]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:00.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: Detected new or changed devices on smithi174 2024-02-07T12:18:00.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:00.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:00.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:00.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi174 to 6432M 2024-02-07T12:18:00.430 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:00.430 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:00.430 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:00.430 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:00.430 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:00.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: Detected new or changed devices on smithi174 2024-02-07T12:18:00.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi174 to 6432M 2024-02-07T12:18:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:01 smithi184 ceph-mon[27098]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:18:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:01.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:01 smithi174 ceph-mon[24580]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:01.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:01.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:18:01.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:02 smithi184 ceph-mon[27098]: Deploying daemon osd.2 on smithi174 2024-02-07T12:18:02.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:02 smithi174 ceph-mon[24580]: Deploying daemon osd.2 on smithi174 2024-02-07T12:18:03.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:03 smithi174 ceph-mon[24580]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:03 smithi184 ceph-mon[27098]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:05.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:05 smithi174 ceph-mon[24580]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:05.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:05 smithi184 ceph-mon[27098]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:07.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:07 smithi174 ceph-mon[24580]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:07 smithi184 ceph-mon[27098]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:08 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:08 smithi174 ceph-mon[24580]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:08 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:08 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:08 smithi184 ceph-mon[27098]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:08 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:11.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:11 smithi174 ceph-mon[24580]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:11 smithi184 ceph-mon[27098]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:12 smithi184 ceph-mon[27098]: from='osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:18:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:12.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:12.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:12.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:12.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:12 smithi174 ceph-mon[24580]: from='osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:18:12.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:12.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:12.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:12.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:12.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:13.010 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 2 on host 'smithi174' 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: from='osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: osdmap e17: 3 total, 2 up, 3 in 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: from='osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:18:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:18:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: from='osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:18:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: osdmap e17: 3 total, 2 up, 3 in 2024-02-07T12:18:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:18:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: from='osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:18:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:13.620 DEBUG:teuthology.orchestra.run.smithi174:osd.2> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.2.service 2024-02-07T12:18:13.622 INFO:tasks.cephadm:Deploying osd.3 on smithi174 with /dev/vg_nvme/lv_1... 2024-02-07T12:18:13.622 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-02-07T12:18:14.355 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:14 smithi174 ceph-mon[24580]: from='osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-02-07T12:18:14.355 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:14 smithi174 ceph-mon[24580]: osdmap e18: 3 total, 2 up, 3 in 2024-02-07T12:18:14.355 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:14 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:18:14.356 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:14 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:18:14.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:14 smithi184 ceph-mon[27098]: from='osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-02-07T12:18:14.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:14 smithi184 ceph-mon[27098]: osdmap e18: 3 total, 2 up, 3 in 2024-02-07T12:18:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:14 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:18:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:14 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:18:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: purged_snaps scrub starts 2024-02-07T12:18:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: purged_snaps scrub ok 2024-02-07T12:18:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846] boot 2024-02-07T12:18:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: osdmap e19: 3 total, 3 up, 3 in 2024-02-07T12:18:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:18:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-07T12:18:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-02-07T12:18:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: osdmap e20: 3 total, 3 up, 3 in 2024-02-07T12:18:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:15 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: purged_snaps scrub starts 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: purged_snaps scrub ok 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: osd.2 [v2:172.21.15.174:6818/2063842846,v1:172.21.15.174:6819/2063842846] boot 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: osdmap e19: 3 total, 3 up, 3 in 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: osdmap e20: 3 total, 3 up, 3 in 2024-02-07T12:18:15.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:15 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-02-07T12:18:16.441 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-02-07T12:18:16.441 INFO:teuthology.orchestra.run.smithi174.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-02-07T12:18:16.441 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10+0 records in 2024-02-07T12:18:16.441 INFO:teuthology.orchestra.run.smithi174.stderr:10+0 records out 2024-02-07T12:18:16.441 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125518 s, 835 MB/s 2024-02-07T12:18:16.441 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping successful for: 2024-02-07T12:18:16.748 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:18:16 smithi174 sudo[42936]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:18:16.748 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:18:16 smithi174 sudo[42936]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:16.748 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:18:16 smithi174 sudo[42936]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:17.016 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:18:16 smithi174 sudo[42944]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:18:17.016 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:18:16 smithi174 sudo[42944]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:17.016 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:18:16 smithi174 sudo[42944]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:17.090 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi174:vg_nvme/lv_1 2024-02-07T12:18:17.280 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[42947]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:18:17.280 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[42947]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:17.281 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[42947]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:17.281 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:17 smithi174 ceph-mon[24580]: pgmap v70: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:17.281 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:17 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-02-07T12:18:17.281 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:17 smithi174 ceph-mon[24580]: osdmap e21: 3 total, 3 up, 3 in 2024-02-07T12:18:17.281 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:17 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:17 smithi184 ceph-mon[27098]: pgmap v70: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:17 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-02-07T12:18:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:17 smithi184 ceph-mon[27098]: osdmap e21: 3 total, 3 up, 3 in 2024-02-07T12:18:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:17 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:17.533 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[42980]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:18:17.534 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[42980]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:17.534 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[42980]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:17.796 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[43013]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:18:17.796 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[43013]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:17.796 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[43013]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:18.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:18 smithi174 sudo[43038]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:18:18.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:18 smithi174 sudo[43038]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:18.047 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[43035]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:18:18.047 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[43035]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:18.047 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:18:17 smithi174 sudo[43035]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:18.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:18 smithi174 sudo[43038]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:18.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:18 smithi174 sudo[43071]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:18:18.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:18 smithi174 sudo[43071]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:18.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:18 smithi174 sudo[43071]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:18.554 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:18 smithi184 ceph-mon[27098]: osdmap e22: 3 total, 3 up, 3 in 2024-02-07T12:18:18.554 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:18 smithi184 ceph-mon[27098]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:18:18.554 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:18 smithi184 sudo[29325]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-02-07T12:18:18.554 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:18 smithi184 sudo[29325]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:18.554 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:18 smithi184 sudo[29325]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:18.589 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:18 smithi174 ceph-mon[24580]: osdmap e22: 3 total, 3 up, 3 in 2024-02-07T12:18:18.589 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:18 smithi174 ceph-mon[24580]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:18:18.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:18 smithi184 sudo[29328]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-02-07T12:18:18.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:18 smithi184 sudo[29328]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-02-07T12:18:18.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:18 smithi184 sudo[29328]: pam_unix(sudo:session): session closed for user root 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: pgmap v73: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: mgrmap e19: smithi174.whsozo(active, since 2m), standbys: smithi184.prgadw 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:18:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:18:19.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:19 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: pgmap v73: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: mgrmap e19: smithi174.whsozo(active, since 2m), standbys: smithi184.prgadw 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:18:19.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:19 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:20.621 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:20 smithi174 ceph-mon[24580]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:20.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:20 smithi184 ceph-mon[27098]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:22.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:22.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:22 smithi174 ceph-mon[24580]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:22.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:22 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2489113075' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8b9a6dc7-f3e0-44b7-a60f-f6bd8f230df4"}]: dispatch 2024-02-07T12:18:22.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:22 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2489113075' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8b9a6dc7-f3e0-44b7-a60f-f6bd8f230df4"}]': finished 2024-02-07T12:18:22.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:22 smithi174 ceph-mon[24580]: osdmap e23: 4 total, 3 up, 4 in 2024-02-07T12:18:22.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:22 smithi184 ceph-mon[27098]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:22 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2489113075' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8b9a6dc7-f3e0-44b7-a60f-f6bd8f230df4"}]: dispatch 2024-02-07T12:18:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:22 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2489113075' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8b9a6dc7-f3e0-44b7-a60f-f6bd8f230df4"}]': finished 2024-02-07T12:18:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:22 smithi184 ceph-mon[27098]: osdmap e23: 4 total, 3 up, 4 in 2024-02-07T12:18:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:23.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:23 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2973414528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:18:23.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:23 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2973414528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:18:24.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:24 smithi174 ceph-mon[24580]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-02-07T12:18:24.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:24 smithi184 ceph-mon[27098]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-02-07T12:18:27.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:27 smithi174 ceph-mon[24580]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-02-07T12:18:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:27 smithi184 ceph-mon[27098]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-02-07T12:18:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: Detected new or changed devices on smithi174 2024-02-07T12:18:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi174 to 4288M 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:28 smithi184 ceph-mon[27098]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: Detected new or changed devices on smithi174 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi174 to 4288M 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:29.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:29.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:29.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:28 smithi174 ceph-mon[24580]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:18:30.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:29 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:18:30.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:29 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:30.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:29 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:18:30.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:29 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:30.984 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:30 smithi174 ceph-mon[24580]: Deploying daemon osd.3 on smithi174 2024-02-07T12:18:30.985 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:30 smithi174 ceph-mon[24580]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:18:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:30 smithi184 ceph-mon[27098]: Deploying daemon osd.3 on smithi174 2024-02-07T12:18:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:30 smithi184 ceph-mon[27098]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:18:33.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:33 smithi174 ceph-mon[24580]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:18:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:33 smithi184 ceph-mon[27098]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-02-07T12:18:35.288 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:35 smithi174 ceph-mon[24580]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-02-07T12:18:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:35 smithi184 ceph-mon[27098]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-02-07T12:18:36.413 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:36 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:36.413 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:36 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:36.413 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:36 smithi174 ceph-mon[24580]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:36.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:36 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:36.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:36 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:36.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:36 smithi184 ceph-mon[27098]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:37.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:37 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:37.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:37.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:37.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:37.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:37 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:38.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:38 smithi184 ceph-mon[27098]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:38.711 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:38 smithi174 ceph-mon[24580]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:40.281 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 3 on host 'smithi174' 2024-02-07T12:18:41.203 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:41 smithi174 ceph-mon[24580]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:41.203 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:41 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:41.203 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:41 smithi174 ceph-mon[24580]: from='osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:18:41.225 DEBUG:teuthology.orchestra.run.smithi174:osd.3> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.3.service 2024-02-07T12:18:41.227 INFO:tasks.cephadm:Deploying osd.4 on smithi184 with /dev/vg_nvme/lv_4... 2024-02-07T12:18:41.228 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-02-07T12:18:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:41 smithi184 ceph-mon[27098]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:41 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:41 smithi184 ceph-mon[27098]: from='osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:18:42.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:42 smithi174 ceph-mon[24580]: from='osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:18:42.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:42 smithi174 ceph-mon[24580]: osdmap e24: 4 total, 3 up, 4 in 2024-02-07T12:18:42.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:42 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:42.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:42 smithi174 ceph-mon[24580]: from='osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:18:42.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:42 smithi174 ceph-mon[24580]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:42.360 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:18:42 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[45562]: 2024-02-07T12:18:42.284+0000 7f1d62da6700 -1 osd.3 0 waiting for initial osdmap 2024-02-07T12:18:42.361 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:18:42 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[45562]: 2024-02-07T12:18:42.289+0000 7f1d5df3e700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:18:42.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:42 smithi184 ceph-mon[27098]: from='osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:18:42.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:42 smithi184 ceph-mon[27098]: osdmap e24: 4 total, 3 up, 4 in 2024-02-07T12:18:42.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:42 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:42.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:42 smithi184 ceph-mon[27098]: from='osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:18:42.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:42 smithi184 ceph-mon[27098]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:43.135 INFO:teuthology.orchestra.run.smithi184.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-02-07T12:18:43.135 INFO:teuthology.orchestra.run.smithi184.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-02-07T12:18:43.135 INFO:teuthology.orchestra.run.smithi184.stderr: stderr: 10+0 records in 2024-02-07T12:18:43.135 INFO:teuthology.orchestra.run.smithi184.stderr:10+0 records out 2024-02-07T12:18:43.135 INFO:teuthology.orchestra.run.smithi184.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011418 s, 918 MB/s 2024-02-07T12:18:43.135 INFO:teuthology.orchestra.run.smithi184.stderr:--> Zapping successful for: 2024-02-07T12:18:43.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:43 smithi184 ceph-mon[27098]: from='osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-02-07T12:18:43.400 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:43 smithi184 ceph-mon[27098]: osdmap e25: 4 total, 3 up, 4 in 2024-02-07T12:18:43.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:43 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:43.401 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:43 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:43 smithi174 ceph-mon[24580]: from='osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-02-07T12:18:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:43 smithi174 ceph-mon[24580]: osdmap e25: 4 total, 3 up, 4 in 2024-02-07T12:18:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:43 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:43.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:43 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:43.868 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi184:vg_nvme/lv_4 2024-02-07T12:18:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:44 smithi184 ceph-mon[27098]: purged_snaps scrub starts 2024-02-07T12:18:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:44 smithi184 ceph-mon[27098]: purged_snaps scrub ok 2024-02-07T12:18:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:44 smithi184 ceph-mon[27098]: osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276] boot 2024-02-07T12:18:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:44 smithi184 ceph-mon[27098]: osdmap e26: 4 total, 4 up, 4 in 2024-02-07T12:18:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:44.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:44 smithi184 ceph-mon[27098]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:44.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:44 smithi174 ceph-mon[24580]: purged_snaps scrub starts 2024-02-07T12:18:44.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:44 smithi174 ceph-mon[24580]: purged_snaps scrub ok 2024-02-07T12:18:44.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:44 smithi174 ceph-mon[24580]: osd.3 [v2:172.21.15.174:6826/1389980276,v1:172.21.15.174:6827/1389980276] boot 2024-02-07T12:18:44.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:44 smithi174 ceph-mon[24580]: osdmap e26: 4 total, 4 up, 4 in 2024-02-07T12:18:44.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:18:44.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:44 smithi174 ceph-mon[24580]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-02-07T12:18:45.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:45 smithi174 ceph-mon[24580]: osdmap e27: 4 total, 4 up, 4 in 2024-02-07T12:18:45.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:45 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:45.561 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:45 smithi184 ceph-mon[27098]: osdmap e27: 4 total, 4 up, 4 in 2024-02-07T12:18:45.561 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:45 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:46.542 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:46 smithi184 ceph-mon[27098]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi184:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:46.542 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:18:46.542 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:18:46.542 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:46.542 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:46 smithi184 ceph-mon[27098]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:46.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:46 smithi174 ceph-mon[24580]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi184:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:18:46.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:18:46.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:18:46.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:46.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:46 smithi174 ceph-mon[24580]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:48.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi174 to 3216M 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:48.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:48.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/1799698005' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6700226-b7c0-48c6-b600-c3d14046351d"}]: dispatch 2024-02-07T12:18:48.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6700226-b7c0-48c6-b600-c3d14046351d"}]: dispatch 2024-02-07T12:18:48.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f6700226-b7c0-48c6-b600-c3d14046351d"}]': finished 2024-02-07T12:18:48.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: osdmap e28: 5 total, 4 up, 5 in 2024-02-07T12:18:48.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:18:48.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:48.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:48 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/3666449701' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:18:48.712 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi174 to 3216M 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/1799698005' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6700226-b7c0-48c6-b600-c3d14046351d"}]: dispatch 2024-02-07T12:18:48.713 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6700226-b7c0-48c6-b600-c3d14046351d"}]: dispatch 2024-02-07T12:18:48.714 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f6700226-b7c0-48c6-b600-c3d14046351d"}]': finished 2024-02-07T12:18:48.714 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: osdmap e28: 5 total, 4 up, 5 in 2024-02-07T12:18:48.714 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:18:48.714 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:48.714 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:48 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/3666449701' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:18:51.158 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:51 smithi184 ceph-mon[27098]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:51.496 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:51 smithi174 ceph-mon[24580]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:52.554 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:52 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:52.555 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:52 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:52.555 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:52 smithi184 ceph-mon[27098]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:52.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:52 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:52.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:52 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:52.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:52 smithi174 ceph-mon[24580]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:53.799 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:53 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:18:53.799 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:53 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:54.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:53 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:18:54.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:53 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:54.918 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:54 smithi184 ceph-mon[27098]: Deploying daemon osd.4 on smithi184 2024-02-07T12:18:54.919 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:54 smithi184 ceph-mon[27098]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:55.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:54 smithi174 ceph-mon[24580]: Deploying daemon osd.4 on smithi184 2024-02-07T12:18:55.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:54 smithi174 ceph-mon[24580]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:57.274 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:57 smithi184 ceph-mon[27098]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:57.443 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:57 smithi174 ceph-mon[24580]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:59.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:59 smithi184 ceph-mon[27098]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:59.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:59 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:59.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:59 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:59.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:59 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:59.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:18:59 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:18:59.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:59 smithi174 ceph-mon[24580]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:18:59.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:59 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:18:59.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:59 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:18:59.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:59 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:18:59.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:18:59 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:01.128 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:01 smithi184 ceph-mon[27098]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:19:01.426 INFO:teuthology.orchestra.run.smithi184.stdout:Created osd(s) 4 on host 'smithi184' 2024-02-07T12:19:01.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:01 smithi174 ceph-mon[24580]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:19:01.961 DEBUG:teuthology.orchestra.run.smithi184:osd.4> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.4.service 2024-02-07T12:19:01.964 INFO:tasks.cephadm:Deploying osd.5 on smithi184 with /dev/vg_nvme/lv_3... 2024-02-07T12:19:01.964 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-02-07T12:19:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:02 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:02 smithi184 ceph-mon[27098]: from='osd.4 [v2:172.21.15.184:6800/764775003,v1:172.21.15.184:6801/764775003]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:19:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:02 smithi184 ceph-mon[27098]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:19:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:02 smithi184 ceph-mon[27098]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:19:02.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:02 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:02.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:02 smithi174 ceph-mon[24580]: from='osd.4 [v2:172.21.15.184:6800/764775003,v1:172.21.15.184:6801/764775003]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:19:02.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:02 smithi174 ceph-mon[24580]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:19:02.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:02 smithi174 ceph-mon[24580]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:19:03.691 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:03 smithi184 ceph-mon[27098]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:19:03.691 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:03 smithi184 ceph-mon[27098]: osdmap e29: 5 total, 4 up, 5 in 2024-02-07T12:19:03.691 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:03 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:19:03.691 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:03 smithi184 ceph-mon[27098]: from='osd.4 [v2:172.21.15.184:6800/764775003,v1:172.21.15.184:6801/764775003]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:03.691 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:03 smithi184 ceph-mon[27098]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:03.691 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:03 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:03.691 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:19:03 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[31245]: 2024-02-07T12:19:03.436+0000 7efd47d66700 -1 osd.4 0 waiting for initial osdmap 2024-02-07T12:19:03.692 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:19:03 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[31245]: 2024-02-07T12:19:03.440+0000 7efd426fd700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:19:03.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:03 smithi174 ceph-mon[24580]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:19:03.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:03 smithi174 ceph-mon[24580]: osdmap e29: 5 total, 4 up, 5 in 2024-02-07T12:19:03.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:03 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:19:03.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:03 smithi174 ceph-mon[24580]: from='osd.4 [v2:172.21.15.184:6800/764775003,v1:172.21.15.184:6801/764775003]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:03.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:03 smithi174 ceph-mon[24580]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:03.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:03 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:04.664 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:04 smithi184 ceph-mon[27098]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]': finished 2024-02-07T12:19:04.664 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:04 smithi184 ceph-mon[27098]: osdmap e30: 5 total, 4 up, 5 in 2024-02-07T12:19:04.664 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:04 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:19:04.665 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:04 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:19:04.665 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:04 smithi184 ceph-mon[27098]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:19:04.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:04 smithi174 ceph-mon[24580]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]': finished 2024-02-07T12:19:04.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:04 smithi174 ceph-mon[24580]: osdmap e30: 5 total, 4 up, 5 in 2024-02-07T12:19:04.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:04 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:19:04.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:04 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:19:04.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:04 smithi174 ceph-mon[24580]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-02-07T12:19:05.039 INFO:teuthology.orchestra.run.smithi184.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-02-07T12:19:05.039 INFO:teuthology.orchestra.run.smithi184.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-02-07T12:19:05.039 INFO:teuthology.orchestra.run.smithi184.stderr: stderr: 10+0 records in 2024-02-07T12:19:05.039 INFO:teuthology.orchestra.run.smithi184.stderr:10+0 records out 2024-02-07T12:19:05.039 INFO:teuthology.orchestra.run.smithi184.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135428 s, 774 MB/s 2024-02-07T12:19:05.039 INFO:teuthology.orchestra.run.smithi184.stderr:--> Zapping successful for: 2024-02-07T12:19:05.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:05 smithi184 ceph-mon[27098]: purged_snaps scrub starts 2024-02-07T12:19:05.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:05 smithi184 ceph-mon[27098]: purged_snaps scrub ok 2024-02-07T12:19:05.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:05 smithi184 ceph-mon[27098]: osd.4 [v2:172.21.15.184:6800/764775003,v1:172.21.15.184:6801/764775003] boot 2024-02-07T12:19:05.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:05 smithi184 ceph-mon[27098]: osdmap e31: 5 total, 5 up, 5 in 2024-02-07T12:19:05.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:05 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:19:05.769 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi184:vg_nvme/lv_3 2024-02-07T12:19:05.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:05 smithi174 ceph-mon[24580]: purged_snaps scrub starts 2024-02-07T12:19:05.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:05 smithi174 ceph-mon[24580]: purged_snaps scrub ok 2024-02-07T12:19:05.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:05 smithi174 ceph-mon[24580]: osd.4 [v2:172.21.15.184:6800/764775003,v1:172.21.15.184:6801/764775003] boot 2024-02-07T12:19:05.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:05 smithi174 ceph-mon[24580]: osdmap e31: 5 total, 5 up, 5 in 2024-02-07T12:19:05.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:05 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:19:06.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:06 smithi184 ceph-mon[27098]: osdmap e32: 5 total, 5 up, 5 in 2024-02-07T12:19:06.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:06 smithi184 ceph-mon[27098]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:06.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:06 smithi174 ceph-mon[24580]: osdmap e32: 5 total, 5 up, 5 in 2024-02-07T12:19:06.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:06 smithi174 ceph-mon[24580]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: osdmap e33: 5 total, 5 up, 5 in 2024-02-07T12:19:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: osdmap e33: 5 total, 5 up, 5 in 2024-02-07T12:19:07.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:07.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:07.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:07.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:07.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:07.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:07.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:07.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:08 smithi174 ceph-mon[24580]: Detected new or changed devices on smithi184 2024-02-07T12:19:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:08 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi184 to 15936M 2024-02-07T12:19:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:08 smithi174 ceph-mon[24580]: osdmap e34: 5 total, 5 up, 5 in 2024-02-07T12:19:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:08 smithi174 ceph-mon[24580]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-02-07T12:19:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:08 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:08 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:08.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:08 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:08.836 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:08 smithi184 ceph-mon[27098]: Detected new or changed devices on smithi184 2024-02-07T12:19:08.836 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:08 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi184 to 15936M 2024-02-07T12:19:08.836 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:08 smithi184 ceph-mon[27098]: osdmap e34: 5 total, 5 up, 5 in 2024-02-07T12:19:08.836 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:08 smithi184 ceph-mon[27098]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-02-07T12:19:08.836 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:08 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:08.837 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:08 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:08.837 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:08 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:09.786 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:09 smithi184 ceph-mon[27098]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi184:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:09.787 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:09 smithi184 ceph-mon[27098]: osdmap e35: 5 total, 5 up, 5 in 2024-02-07T12:19:09.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:09 smithi174 ceph-mon[24580]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi184:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:09.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:09 smithi174 ceph-mon[24580]: osdmap e35: 5 total, 5 up, 5 in 2024-02-07T12:19:10.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:10 smithi174 ceph-mon[24580]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-02-07T12:19:10.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:10 smithi184 ceph-mon[27098]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-02-07T12:19:11.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:11 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/3689334068' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0732d783-5be3-4429-8a53-b575b0d5f880"}]: dispatch 2024-02-07T12:19:11.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:11 smithi174 ceph-mon[24580]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0732d783-5be3-4429-8a53-b575b0d5f880"}]: dispatch 2024-02-07T12:19:11.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:11 smithi174 ceph-mon[24580]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0732d783-5be3-4429-8a53-b575b0d5f880"}]': finished 2024-02-07T12:19:11.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:11 smithi174 ceph-mon[24580]: osdmap e36: 6 total, 5 up, 6 in 2024-02-07T12:19:11.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:11.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:11 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/3189295569' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:11.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:11 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/3689334068' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0732d783-5be3-4429-8a53-b575b0d5f880"}]: dispatch 2024-02-07T12:19:11.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:11 smithi184 ceph-mon[27098]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0732d783-5be3-4429-8a53-b575b0d5f880"}]: dispatch 2024-02-07T12:19:11.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:11 smithi184 ceph-mon[27098]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0732d783-5be3-4429-8a53-b575b0d5f880"}]': finished 2024-02-07T12:19:11.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:11 smithi184 ceph-mon[27098]: osdmap e36: 6 total, 5 up, 6 in 2024-02-07T12:19:11.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:11.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:11 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/3189295569' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:12.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:12 smithi174 ceph-mon[24580]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-02-07T12:19:12.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:19:12.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:19:12.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:12 smithi184 ceph-mon[27098]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-02-07T12:19:12.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:19:12.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:19:15.399 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:15 smithi184 ceph-mon[27098]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-02-07T12:19:15.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:15 smithi174 ceph-mon[24580]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-02-07T12:19:17.360 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:17 smithi184 ceph-mon[27098]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:17.360 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:17 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:19:17.360 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:17 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:17.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:17 smithi174 ceph-mon[24580]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:17.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:17 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:19:17.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:17 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:18.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:18 smithi184 ceph-mon[27098]: Deploying daemon osd.5 on smithi184 2024-02-07T12:19:18.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:18 smithi174 ceph-mon[24580]: Deploying daemon osd.5 on smithi184 2024-02-07T12:19:19.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:19 smithi184 ceph-mon[27098]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:19.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:19 smithi174 ceph-mon[24580]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:21.174 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:21 smithi184 ceph-mon[27098]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:21.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:21 smithi174 ceph-mon[24580]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:22.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:22.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:22 smithi184 ceph-mon[27098]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:22.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:22.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:22.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:22.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:22.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:22 smithi174 ceph-mon[24580]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:24.122 INFO:teuthology.orchestra.run.smithi184.stdout:Created osd(s) 5 on host 'smithi184' 2024-02-07T12:19:24.663 DEBUG:teuthology.orchestra.run.smithi184:osd.5> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.5.service 2024-02-07T12:19:24.666 INFO:tasks.cephadm:Deploying osd.6 on smithi184 with /dev/vg_nvme/lv_2... 2024-02-07T12:19:24.666 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-02-07T12:19:25.445 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:25 smithi184 ceph-mon[27098]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:25.445 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:25 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:25.445 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:25 smithi184 ceph-mon[27098]: from='osd.5 [v2:172.21.15.184:6808/559985402,v1:172.21.15.184:6809/559985402]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:19:25.445 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:25 smithi184 ceph-mon[27098]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:19:25.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:25 smithi174 ceph-mon[24580]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:25.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:25 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:25.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:25 smithi174 ceph-mon[24580]: from='osd.5 [v2:172.21.15.184:6808/559985402,v1:172.21.15.184:6809/559985402]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:19:25.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:25 smithi174 ceph-mon[24580]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:19:26.381 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:26 smithi184 ceph-mon[27098]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:19:26.381 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:26 smithi184 ceph-mon[27098]: osdmap e37: 6 total, 5 up, 6 in 2024-02-07T12:19:26.382 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:26 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:26.382 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:26 smithi184 ceph-mon[27098]: from='osd.5 [v2:172.21.15.184:6808/559985402,v1:172.21.15.184:6809/559985402]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:26.382 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:26 smithi184 ceph-mon[27098]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:26.382 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:19:26 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[34229]: 2024-02-07T12:19:26.132+0000 7f36d0cc1700 -1 osd.5 0 waiting for initial osdmap 2024-02-07T12:19:26.382 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:19:26 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[34229]: 2024-02-07T12:19:26.137+0000 7f36cae57700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:19:26.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:26 smithi174 ceph-mon[24580]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:19:26.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:26 smithi174 ceph-mon[24580]: osdmap e37: 6 total, 5 up, 6 in 2024-02-07T12:19:26.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:26 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:26.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:26 smithi174 ceph-mon[24580]: from='osd.5 [v2:172.21.15.184:6808/559985402,v1:172.21.15.184:6809/559985402]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:26.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:26 smithi174 ceph-mon[24580]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:27.262 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:27 smithi184 ceph-mon[27098]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:27.262 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:27 smithi184 ceph-mon[27098]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]': finished 2024-02-07T12:19:27.262 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:27 smithi184 ceph-mon[27098]: osdmap e38: 6 total, 5 up, 6 in 2024-02-07T12:19:27.262 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:27.262 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:27 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:27.377 INFO:teuthology.orchestra.run.smithi184.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-02-07T12:19:27.377 INFO:teuthology.orchestra.run.smithi184.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-02-07T12:19:27.378 INFO:teuthology.orchestra.run.smithi184.stderr: stderr: 10+0 records in 2024-02-07T12:19:27.378 INFO:teuthology.orchestra.run.smithi184.stderr:10+0 records out 2024-02-07T12:19:27.378 INFO:teuthology.orchestra.run.smithi184.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0140932 s, 744 MB/s 2024-02-07T12:19:27.378 INFO:teuthology.orchestra.run.smithi184.stderr:--> Zapping successful for: 2024-02-07T12:19:27.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:27 smithi174 ceph-mon[24580]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:27 smithi174 ceph-mon[24580]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]': finished 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:27 smithi174 ceph-mon[24580]: osdmap e38: 6 total, 5 up, 6 in 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:27.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:27 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:27.974 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi184:vg_nvme/lv_2 2024-02-07T12:19:28.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:28 smithi184 ceph-mon[27098]: purged_snaps scrub starts 2024-02-07T12:19:28.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:28 smithi184 ceph-mon[27098]: purged_snaps scrub ok 2024-02-07T12:19:28.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:28.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:28 smithi184 ceph-mon[27098]: osd.5 [v2:172.21.15.184:6808/559985402,v1:172.21.15.184:6809/559985402] boot 2024-02-07T12:19:28.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:28 smithi184 ceph-mon[27098]: osdmap e39: 6 total, 6 up, 6 in 2024-02-07T12:19:28.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:28.320 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:28 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:28.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:28 smithi174 ceph-mon[24580]: purged_snaps scrub starts 2024-02-07T12:19:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:28 smithi174 ceph-mon[24580]: purged_snaps scrub ok 2024-02-07T12:19:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:28 smithi174 ceph-mon[24580]: osd.5 [v2:172.21.15.184:6808/559985402,v1:172.21.15.184:6809/559985402] boot 2024-02-07T12:19:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:28 smithi174 ceph-mon[24580]: osdmap e39: 6 total, 6 up, 6 in 2024-02-07T12:19:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:19:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:28 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:29.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:29 smithi184 ceph-mon[27098]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:29.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:29 smithi184 ceph-mon[27098]: osdmap e40: 6 total, 6 up, 6 in 2024-02-07T12:19:29.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:29 smithi184 ceph-mon[27098]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:19:29.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:29 smithi174 ceph-mon[24580]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-02-07T12:19:29.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:29 smithi174 ceph-mon[24580]: osdmap e40: 6 total, 6 up, 6 in 2024-02-07T12:19:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:29 smithi174 ceph-mon[24580]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:19:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:30 smithi184 ceph-mon[27098]: osdmap e41: 6 total, 6 up, 6 in 2024-02-07T12:19:30.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:30 smithi174 ceph-mon[24580]: osdmap e41: 6 total, 6 up, 6 in 2024-02-07T12:19:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:31 smithi184 ceph-mon[27098]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:31 smithi184 ceph-mon[27098]: osdmap e42: 6 total, 6 up, 6 in 2024-02-07T12:19:31.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:31 smithi174 ceph-mon[24580]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:31.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:31 smithi174 ceph-mon[24580]: osdmap e42: 6 total, 6 up, 6 in 2024-02-07T12:19:32.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: osdmap e43: 6 total, 6 up, 6 in 2024-02-07T12:19:32.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:32.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:32.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:32.414 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:32 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:32.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: osdmap e43: 6 total, 6 up, 6 in 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:32.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:32 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:33.412 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:33 smithi184 ceph-mon[27098]: Detected new or changed devices on smithi184 2024-02-07T12:19:33.412 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:33 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi184 to 7968M 2024-02-07T12:19:33.412 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:33 smithi184 ceph-mon[27098]: from='client.24185 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi184:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:33.412 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:33 smithi184 ceph-mon[27098]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:33.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:33 smithi174 ceph-mon[24580]: Detected new or changed devices on smithi184 2024-02-07T12:19:33.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:33 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi184 to 7968M 2024-02-07T12:19:33.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:33 smithi174 ceph-mon[24580]: from='client.24185 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi184:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:33.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:33 smithi174 ceph-mon[24580]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:34 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/1986151166' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8aa24f1-32f7-46ea-b44e-490ffeeacbd7"}]: dispatch 2024-02-07T12:19:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:34 smithi184 ceph-mon[27098]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8aa24f1-32f7-46ea-b44e-490ffeeacbd7"}]: dispatch 2024-02-07T12:19:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:34 smithi184 ceph-mon[27098]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d8aa24f1-32f7-46ea-b44e-490ffeeacbd7"}]': finished 2024-02-07T12:19:34.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:34 smithi184 ceph-mon[27098]: osdmap e44: 7 total, 6 up, 7 in 2024-02-07T12:19:34.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:34 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:34.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:34 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/1986151166' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8aa24f1-32f7-46ea-b44e-490ffeeacbd7"}]: dispatch 2024-02-07T12:19:34.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:34 smithi174 ceph-mon[24580]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d8aa24f1-32f7-46ea-b44e-490ffeeacbd7"}]: dispatch 2024-02-07T12:19:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:34 smithi174 ceph-mon[24580]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d8aa24f1-32f7-46ea-b44e-490ffeeacbd7"}]': finished 2024-02-07T12:19:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:34 smithi174 ceph-mon[24580]: osdmap e44: 7 total, 6 up, 7 in 2024-02-07T12:19:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:34 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:35 smithi184 ceph-mon[27098]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-02-07T12:19:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:35 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/3120435464' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:35 smithi184 ceph-mon[27098]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-07T12:19:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:35 smithi184 ceph-mon[27098]: Cluster is now healthy 2024-02-07T12:19:35.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:35 smithi174 ceph-mon[24580]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-02-07T12:19:35.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:35 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/3120435464' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:35.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:35 smithi174 ceph-mon[24580]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-07T12:19:35.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:35 smithi174 ceph-mon[24580]: Cluster is now healthy 2024-02-07T12:19:37.409 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:37 smithi184 ceph-mon[27098]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:19:37.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:37 smithi174 ceph-mon[24580]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:19:39.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:39 smithi184 ceph-mon[27098]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:19:39.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:39 smithi174 ceph-mon[24580]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:19:40.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:19:40.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:40.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:19:40.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:41.445 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:41 smithi184 ceph-mon[27098]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-02-07T12:19:41.446 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:41 smithi184 ceph-mon[27098]: Deploying daemon osd.6 on smithi184 2024-02-07T12:19:41.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:41 smithi174 ceph-mon[24580]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-02-07T12:19:41.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:41 smithi174 ceph-mon[24580]: Deploying daemon osd.6 on smithi184 2024-02-07T12:19:43.428 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:43 smithi184 ceph-mon[27098]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:19:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:43 smithi174 ceph-mon[24580]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:19:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:45 smithi184 ceph-mon[27098]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:45.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:45 smithi174 ceph-mon[24580]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:46.509 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:46.509 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:46.509 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:46.509 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:46 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:46.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:46.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:46.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:46.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:46 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:47.439 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:47 smithi184 ceph-mon[27098]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:47.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:47 smithi174 ceph-mon[24580]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:47.904 INFO:teuthology.orchestra.run.smithi184.stdout:Created osd(s) 6 on host 'smithi184' 2024-02-07T12:19:48.815 DEBUG:teuthology.orchestra.run.smithi184:osd.6> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.6.service 2024-02-07T12:19:48.818 INFO:tasks.cephadm:Deploying osd.7 on smithi184 with /dev/vg_nvme/lv_1... 2024-02-07T12:19:48.818 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-02-07T12:19:49.156 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:48 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:49.157 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:48 smithi184 ceph-mon[27098]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:49.157 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:48 smithi184 ceph-mon[27098]: from='osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:19:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:48 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:48 smithi174 ceph-mon[24580]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:48 smithi174 ceph-mon[24580]: from='osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:19:50.024 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:49 smithi184 ceph-mon[27098]: from='osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:19:50.025 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:49 smithi184 ceph-mon[27098]: osdmap e45: 7 total, 6 up, 7 in 2024-02-07T12:19:50.025 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:50.025 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:49 smithi184 ceph-mon[27098]: from='osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:50.025 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:50.026 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:19:49 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[37211]: 2024-02-07T12:19:49.911+0000 7f990f268700 -1 osd.6 0 waiting for initial osdmap 2024-02-07T12:19:50.026 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:19:49 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[37211]: 2024-02-07T12:19:49.920+0000 7f9909bff700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:19:50.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:49 smithi174 ceph-mon[24580]: from='osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:19:50.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:49 smithi174 ceph-mon[24580]: osdmap e45: 7 total, 6 up, 7 in 2024-02-07T12:19:50.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:50.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:49 smithi174 ceph-mon[24580]: from='osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:19:50.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:50 smithi184 ceph-mon[27098]: from='osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]': finished 2024-02-07T12:19:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:50 smithi184 ceph-mon[27098]: osdmap e46: 7 total, 6 up, 7 in 2024-02-07T12:19:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:50 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:50 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:50 smithi184 ceph-mon[27098]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:51.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:50 smithi184 ceph-mon[27098]: osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647] boot 2024-02-07T12:19:51.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:50 smithi184 ceph-mon[27098]: osdmap e47: 7 total, 7 up, 7 in 2024-02-07T12:19:51.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:50 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:50 smithi174 ceph-mon[24580]: from='osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]': finished 2024-02-07T12:19:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:50 smithi174 ceph-mon[24580]: osdmap e46: 7 total, 6 up, 7 in 2024-02-07T12:19:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:50 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:50 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:50 smithi174 ceph-mon[24580]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-02-07T12:19:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:50 smithi174 ceph-mon[24580]: osd.6 [v2:172.21.15.184:6816/273946647,v1:172.21.15.184:6817/273946647] boot 2024-02-07T12:19:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:50 smithi174 ceph-mon[24580]: osdmap e47: 7 total, 7 up, 7 in 2024-02-07T12:19:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:50 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:19:51.339 INFO:teuthology.orchestra.run.smithi184.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-02-07T12:19:51.339 INFO:teuthology.orchestra.run.smithi184.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-02-07T12:19:51.339 INFO:teuthology.orchestra.run.smithi184.stderr: stderr: 10+0 records in 2024-02-07T12:19:51.339 INFO:teuthology.orchestra.run.smithi184.stderr:10+0 records out 2024-02-07T12:19:51.339 INFO:teuthology.orchestra.run.smithi184.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0126956 s, 826 MB/s 2024-02-07T12:19:51.340 INFO:teuthology.orchestra.run.smithi184.stderr:--> Zapping successful for: 2024-02-07T12:19:51.952 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch daemon add osd smithi184:vg_nvme/lv_1 2024-02-07T12:19:52.188 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:52 smithi184 ceph-mon[27098]: purged_snaps scrub starts 2024-02-07T12:19:52.188 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:52 smithi184 ceph-mon[27098]: purged_snaps scrub ok 2024-02-07T12:19:52.188 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:52 smithi184 ceph-mon[27098]: osdmap e48: 7 total, 7 up, 7 in 2024-02-07T12:19:52.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:52 smithi174 ceph-mon[24580]: purged_snaps scrub starts 2024-02-07T12:19:52.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:52 smithi174 ceph-mon[24580]: purged_snaps scrub ok 2024-02-07T12:19:52.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:52 smithi174 ceph-mon[24580]: osdmap e48: 7 total, 7 up, 7 in 2024-02-07T12:19:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:53 smithi184 ceph-mon[27098]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:19:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:53 smithi184 ceph-mon[27098]: osdmap e49: 7 total, 7 up, 7 in 2024-02-07T12:19:53.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:53 smithi174 ceph-mon[24580]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:19:53.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:53 smithi174 ceph-mon[24580]: osdmap e49: 7 total, 7 up, 7 in 2024-02-07T12:19:54.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:54 smithi184 ceph-mon[27098]: osdmap e50: 7 total, 7 up, 7 in 2024-02-07T12:19:54.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:54 smithi174 ceph-mon[24580]: osdmap e50: 7 total, 7 up, 7 in 2024-02-07T12:19:55.093 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:55 smithi184 ceph-mon[27098]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:19:55.093 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:55 smithi184 ceph-mon[27098]: osdmap e51: 7 total, 7 up, 7 in 2024-02-07T12:19:55.093 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:55 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:55.093 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:55 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:55.093 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:55 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:55.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:55 smithi174 ceph-mon[24580]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:19:55.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:55 smithi174 ceph-mon[24580]: osdmap e51: 7 total, 7 up, 7 in 2024-02-07T12:19:55.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:55 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-02-07T12:19:55.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:55 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-02-07T12:19:55.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:55 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi184:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:56.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:56.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi184:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:19:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:19:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:57 smithi184 ceph-mon[27098]: Detected new or changed devices on smithi184 2024-02-07T12:19:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:57 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi184 to 5312M 2024-02-07T12:19:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:57 smithi184 ceph-mon[27098]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-02-07T12:19:57.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:57 smithi174 ceph-mon[24580]: Detected new or changed devices on smithi184 2024-02-07T12:19:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:57 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi184 to 5312M 2024-02-07T12:19:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:57 smithi174 ceph-mon[24580]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-02-07T12:19:58.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:58 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/402394537' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d161396-67be-4db2-bc1c-88d810d907b5"}]: dispatch 2024-02-07T12:19:58.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:58 smithi184 ceph-mon[27098]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d161396-67be-4db2-bc1c-88d810d907b5"}]: dispatch 2024-02-07T12:19:58.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:58 smithi184 ceph-mon[27098]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d161396-67be-4db2-bc1c-88d810d907b5"}]': finished 2024-02-07T12:19:58.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:58 smithi184 ceph-mon[27098]: osdmap e52: 8 total, 7 up, 8 in 2024-02-07T12:19:58.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:58 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:19:58.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:58 smithi184 ceph-mon[27098]: from='client.? 172.21.15.184:0/119449416' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:58.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:58 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/402394537' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d161396-67be-4db2-bc1c-88d810d907b5"}]: dispatch 2024-02-07T12:19:58.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:58 smithi174 ceph-mon[24580]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d161396-67be-4db2-bc1c-88d810d907b5"}]: dispatch 2024-02-07T12:19:58.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:58 smithi174 ceph-mon[24580]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d161396-67be-4db2-bc1c-88d810d907b5"}]': finished 2024-02-07T12:19:58.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:58 smithi174 ceph-mon[24580]: osdmap e52: 8 total, 7 up, 8 in 2024-02-07T12:19:58.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:58 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:19:58.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:58 smithi174 ceph-mon[24580]: from='client.? 172.21.15.184:0/119449416' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-02-07T12:19:59.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:19:59 smithi184 ceph-mon[27098]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:19:59.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:19:59 smithi174 ceph-mon[24580]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:20:00.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:00 smithi184 ceph-mon[27098]: overall HEALTH_OK 2024-02-07T12:20:00.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:00 smithi174 ceph-mon[24580]: overall HEALTH_OK 2024-02-07T12:20:01.362 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:01 smithi184 ceph-mon[27098]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-02-07T12:20:01.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:01 smithi174 ceph-mon[24580]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-02-07T12:20:03.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:03 smithi184 ceph-mon[27098]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:20:03.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:03 smithi174 ceph-mon[24580]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-02-07T12:20:04.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:04 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:20:04.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:04 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:04.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:04 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:20:04.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:04 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:05.284 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:05 smithi184 ceph-mon[27098]: Deploying daemon osd.7 on smithi184 2024-02-07T12:20:05.284 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:05 smithi184 ceph-mon[27098]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:20:05.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:05 smithi174 ceph-mon[24580]: Deploying daemon osd.7 on smithi184 2024-02-07T12:20:05.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:05 smithi174 ceph-mon[24580]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-02-07T12:20:07.349 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:07 smithi174 ceph-mon[24580]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:07.359 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:07 smithi184 ceph-mon[27098]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:09.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:09 smithi174 ceph-mon[24580]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:09.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:09 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:09.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:09 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:09.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:09 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:09.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:09 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:09.571 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:09 smithi184 ceph-mon[27098]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:09.571 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:09 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:09.572 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:09 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:09.572 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:09 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:09.572 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:09 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:11.121 INFO:teuthology.orchestra.run.smithi184.stdout:Created osd(s) 7 on host 'smithi184' 2024-02-07T12:20:11.417 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:11 smithi184 ceph-mon[27098]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:11.418 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:11.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:11 smithi174 ceph-mon[24580]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:11.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:11.716 DEBUG:teuthology.orchestra.run.smithi184:osd.7> sudo journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.7.service 2024-02-07T12:20:11.719 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-02-07T12:20:11.719 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd stat -f json 2024-02-07T12:20:12.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:12 smithi174 ceph-mon[24580]: from='osd.7 [v2:172.21.15.184:6824/649846180,v1:172.21.15.184:6825/649846180]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:20:12.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:12 smithi174 ceph-mon[24580]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:20:12.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:20:12.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:20:12.337 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:12 smithi184 ceph-mon[27098]: from='osd.7 [v2:172.21.15.184:6824/649846180,v1:172.21.15.184:6825/649846180]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:20:12.337 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:12 smithi184 ceph-mon[27098]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:20:12.337 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:20:12.337 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:20:13.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-mon[27098]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:13.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-mon[27098]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:20:13.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-mon[27098]: osdmap e53: 8 total, 7 up, 8 in 2024-02-07T12:20:13.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:13.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-mon[27098]: from='osd.7 [v2:172.21.15.184:6824/649846180,v1:172.21.15.184:6825/649846180]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:20:13.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-mon[27098]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:20:13.384 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:13.385 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[40290]: 2024-02-07T12:20:13.242+0000 7fcad6471700 -1 osd.7 0 waiting for initial osdmap 2024-02-07T12:20:13.385 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:20:13 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[40290]: 2024-02-07T12:20:13.249+0000 7fcacee04700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:20:13.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:13 smithi174 ceph-mon[24580]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:13.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:13 smithi174 ceph-mon[24580]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:20:13.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:13 smithi174 ceph-mon[24580]: osdmap e53: 8 total, 7 up, 8 in 2024-02-07T12:20:13.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:13.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:13 smithi174 ceph-mon[24580]: from='osd.7 [v2:172.21.15.184:6824/649846180,v1:172.21.15.184:6825/649846180]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:20:13.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:13 smithi174 ceph-mon[24580]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:20:13.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:13.760 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:20:14.306 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:14 smithi184 ceph-mon[27098]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]': finished 2024-02-07T12:20:14.307 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:14 smithi184 ceph-mon[27098]: osdmap e54: 8 total, 7 up, 8 in 2024-02-07T12:20:14.307 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:14 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:14.307 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:14 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:14.307 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:14 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2765650401' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-07T12:20:14.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:14 smithi174 ceph-mon[24580]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]': finished 2024-02-07T12:20:14.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:14 smithi174 ceph-mon[24580]: osdmap e54: 8 total, 7 up, 8 in 2024-02-07T12:20:14.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:14 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:14.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:14 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:14.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:14 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2765650401' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-07T12:20:14.427 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1707308390,"num_in_osds":8,"osd_in_since":1707308397,"num_remapped_pgs":0} 2024-02-07T12:20:15.420 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:15 smithi184 ceph-mon[27098]: purged_snaps scrub starts 2024-02-07T12:20:15.420 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:15 smithi184 ceph-mon[27098]: purged_snaps scrub ok 2024-02-07T12:20:15.420 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:15 smithi184 ceph-mon[27098]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:15.420 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:15 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:15.420 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:15 smithi184 ceph-mon[27098]: osd.7 [v2:172.21.15.184:6824/649846180,v1:172.21.15.184:6825/649846180] boot 2024-02-07T12:20:15.420 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:15 smithi184 ceph-mon[27098]: osdmap e55: 8 total, 8 up, 8 in 2024-02-07T12:20:15.420 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:15 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:15.428 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd stat -f json 2024-02-07T12:20:15.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:15 smithi174 ceph-mon[24580]: purged_snaps scrub starts 2024-02-07T12:20:15.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:15 smithi174 ceph-mon[24580]: purged_snaps scrub ok 2024-02-07T12:20:15.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:15 smithi174 ceph-mon[24580]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-02-07T12:20:15.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:15 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:15.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:15 smithi174 ceph-mon[24580]: osd.7 [v2:172.21.15.184:6824/649846180,v1:172.21.15.184:6825/649846180] boot 2024-02-07T12:20:15.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:15 smithi174 ceph-mon[24580]: osdmap e55: 8 total, 8 up, 8 in 2024-02-07T12:20:15.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:15 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:20:16.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: osdmap e56: 8 total, 8 up, 8 in 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: osdmap e56: 8 total, 8 up, 8 in 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:20:16.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:20:16.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:20:17.341 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:20:17.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:17 smithi174 ceph-mon[24580]: Detected new or changed devices on smithi184 2024-02-07T12:20:17.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:17 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi184 to 3984M 2024-02-07T12:20:17.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:17 smithi174 ceph-mon[24580]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:17.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:17 smithi174 ceph-mon[24580]: osdmap e57: 8 total, 8 up, 8 in 2024-02-07T12:20:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:17 smithi184 ceph-mon[27098]: Detected new or changed devices on smithi184 2024-02-07T12:20:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:17 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi184 to 3984M 2024-02-07T12:20:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:17 smithi184 ceph-mon[27098]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:17 smithi184 ceph-mon[27098]: osdmap e57: 8 total, 8 up, 8 in 2024-02-07T12:20:17.939 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1707308414,"num_in_osds":8,"osd_in_since":1707308397,"num_remapped_pgs":0} 2024-02-07T12:20:17.939 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd dump --format=json 2024-02-07T12:20:18.152 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:18.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:18 smithi174 ceph-mon[24580]: osdmap e58: 8 total, 8 up, 8 in 2024-02-07T12:20:18.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:18 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2647309996' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-07T12:20:18.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:18 smithi184 ceph-mon[27098]: osdmap e58: 8 total, 8 up, 8 in 2024-02-07T12:20:18.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:18 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2647309996' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-02-07T12:20:19.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:19 smithi174 ceph-mon[24580]: pgmap v169: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-02-07T12:20:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:19 smithi174 ceph-mon[24580]: osdmap e59: 8 total, 8 up, 8 in 2024-02-07T12:20:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:19 smithi184 ceph-mon[27098]: pgmap v169: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-02-07T12:20:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:19 smithi184 ceph-mon[27098]: osdmap e59: 8 total, 8 up, 8 in 2024-02-07T12:20:19.977 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:20:19.977 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":59,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","created":"2024-02-07T12:14:19.506549+0000","modified":"2024-02-07T12:20:18.264682+0000","last_up_change":"2024-02-07T12:20:14.240676+0000","last_in_change":"2024-02-07T12:19:57.278587+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-02-07T12:18:14.333876+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"67efbac6-68ee-4b1d-b17b-30ccdecd5fd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6803","nonce":3253424960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6805","nonce":3253424960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6809","nonce":3253424960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6807","nonce":3253424960}]},"public_addr":"172.21.15.174:6803/3253424960","cluster_addr":"172.21.15.174:6805/3253424960","heartbeat_back_addr":"172.21.15.174:6809/3253424960","heartbeat_front_addr":"172.21.15.174:6807/3253424960","state":["exists","up"]},{"osd":1,"uuid":"648a2717-71c4-4f4b-a47b-1ef7562ce576","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6811","nonce":3157081833}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6813","nonce":3157081833}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6817","nonce":3157081833}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6815","nonce":3157081833}]},"public_addr":"172.21.15.174:6811/3157081833","cluster_addr":"172.21.15.174:6813/3157081833","heartbeat_back_addr":"172.21.15.174:6817/3157081833","heartbeat_front_addr":"172.21.15.174:6815/3157081833","state":["exists","up"]},{"osd":2,"uuid":"c005d20a-a54c-4d99-bf9b-18c11f2accf5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6819","nonce":2063842846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6821","nonce":2063842846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6825","nonce":2063842846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6823","nonce":2063842846}]},"public_addr":"172.21.15.174:6819/2063842846","cluster_addr":"172.21.15.174:6821/2063842846","heartbeat_back_addr":"172.21.15.174:6825/2063842846","heartbeat_front_addr":"172.21.15.174:6823/2063842846","state":["exists","up"]},{"osd":3,"uuid":"8b9a6dc7-f3e0-44b7-a60f-f6bd8f230df4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6827","nonce":1389980276}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6829","nonce":1389980276}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6832","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6833","nonce":1389980276}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6831","nonce":1389980276}]},"public_addr":"172.21.15.174:6827/1389980276","cluster_addr":"172.21.15.174:6829/1389980276","heartbeat_back_addr":"172.21.15.174:6833/1389980276","heartbeat_front_addr":"172.21.15.174:6831/1389980276","state":["exists","up"]},{"osd":4,"uuid":"f6700226-b7c0-48c6-b600-c3d14046351d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6800","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6801","nonce":764775003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6802","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6803","nonce":764775003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6806","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6807","nonce":764775003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6804","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6805","nonce":764775003}]},"public_addr":"172.21.15.184:6801/764775003","cluster_addr":"172.21.15.184:6803/764775003","heartbeat_back_addr":"172.21.15.184:6807/764775003","heartbeat_front_addr":"172.21.15.184:6805/764775003","state":["exists","up"]},{"osd":5,"uuid":"0732d783-5be3-4429-8a53-b575b0d5f880","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6808","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6809","nonce":559985402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6810","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6811","nonce":559985402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6814","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6815","nonce":559985402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6812","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6813","nonce":559985402}]},"public_addr":"172.21.15.184:6809/559985402","cluster_addr":"172.21.15.184:6811/559985402","heartbeat_back_addr":"172.21.15.184:6815/559985402","heartbeat_front_addr":"172.21.15.184:6813/559985402","state":["exists","up"]},{"osd":6,"uuid":"d8aa24f1-32f7-46ea-b44e-490ffeeacbd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6816","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6817","nonce":273946647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6818","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6819","nonce":273946647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6822","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6823","nonce":273946647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6820","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6821","nonce":273946647}]},"public_addr":"172.21.15.184:6817/273946647","cluster_addr":"172.21.15.184:6819/273946647","heartbeat_back_addr":"172.21.15.184:6823/273946647","heartbeat_front_addr":"172.21.15.184:6821/273946647","state":["exists","up"]},{"osd":7,"uuid":"7d161396-67be-4db2-bc1c-88d810d907b5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6824","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6825","nonce":649846180}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6826","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6827","nonce":649846180}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6830","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6831","nonce":649846180}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6828","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6829","nonce":649846180}]},"public_addr":"172.21.15.184:6825/649846180","cluster_addr":"172.21.15.184:6827/649846180","heartbeat_back_addr":"172.21.15.184:6831/649846180","heartbeat_front_addr":"172.21.15.184:6829/649846180","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:17:19.350238+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:17:44.721918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:18:12.247857+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:18:41.409447+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:02.678429+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:25.538675+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:49.235242+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:12.402006+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.174:0/2512161230":"2024-02-08T12:16:11.043144+0000","172.21.15.174:0/4031027938":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6801/3745610189":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6800/3745610189":"2024-02-08T12:16:11.043144+0000","172.21.15.174:0/547150068":"2024-02-08T12:15:13.880259+0000","172.21.15.174:6800/2133101063":"2024-02-08T12:14:44.108968+0000","172.21.15.174:6801/3584666552":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1253517459":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1469259571":"2024-02-08T12:14:44.108968+0000","172.21.15.174:6801/2133101063":"2024-02-08T12:14:44.108968+0000","172.21.15.174:0/1798658282":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6800/3584666552":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/601484952":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1506488865":"2024-02-08T12:14:44.108968+0000","172.21.15.174:0/2510143664":"2024-02-08T12:14:44.108968+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-07T12:20:20.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:20 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2219952511' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:20:20.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:20 smithi174 ceph-mon[24580]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-02-07T12:20:20.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:20 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2219952511' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:20:20.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:20 smithi184 ceph-mon[27098]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-02-07T12:20:20.731 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-02-07T12:18:14.333876+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-02-07T12:20:20.732 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-02-07T12:20:20.944 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:22.741 INFO:teuthology.orchestra.run.smithi174.stdout:pg_num: 1 2024-02-07T12:20:23.422 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:23 smithi174 ceph-mon[24580]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:20:23.422 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:23 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/290062148' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-02-07T12:20:23.446 INFO:tasks.cephadm:Setting up client nodes... 2024-02-07T12:20:23.446 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-02-07T12:20:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:23 smithi184 ceph-mon[27098]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-02-07T12:20:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:23 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/290062148' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-02-07T12:20:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:25 smithi184 ceph-mon[27098]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-02-07T12:20:25.513 INFO:teuthology.orchestra.run.smithi174.stdout:[client.0] 2024-02-07T12:20:25.513 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQCJdcNlsQxGHhAAEGqXSYBYIA3Opq3HhcirOg== 2024-02-07T12:20:25.514 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:25 smithi174 ceph-mon[24580]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-02-07T12:20:26.250 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-02-07T12:20:26.250 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-02-07T12:20:26.250 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-02-07T12:20:26.292 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-02-07T12:20:26.293 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-02-07T12:20:26.293 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph mgr dump --format=json 2024-02-07T12:20:26.356 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:26 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2424093979' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:20:26.356 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:26 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2424093979' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-07T12:20:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:26 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2424093979' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-02-07T12:20:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:26 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2424093979' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-02-07T12:20:26.544 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:27.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:27 smithi174 ceph-mon[24580]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:27.349 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:27 smithi184 ceph-mon[27098]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:28.199 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:20:28.941 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi174.whsozo","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":1130451376},{"type":"v1","addr":"172.21.15.174:6801","nonce":1130451376}]},"active_addr":"172.21.15.174:6801/1130451376","active_change":"2024-02-07T12:16:11.043453+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14226,"name":"smithi184.prgadw","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.174:8443/","prometheus":"http://172.21.15.174:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.174:0","nonce":1011580653}]},{"addrvec":[{"type":"v2","addr":"172.21.15.174:0","nonce":607390536}]},{"addrvec":[{"type":"v2","addr":"172.21.15.174:0","nonce":2418493127}]},{"addrvec":[{"type":"v2","addr":"172.21.15.174:0","nonce":947724195}]}]}} 2024-02-07T12:20:28.948 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-02-07T12:20:28.948 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-02-07T12:20:28.948 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd dump --format=json 2024-02-07T12:20:29.196 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:29.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:29 smithi174 ceph-mon[24580]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:29.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:29 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/1931326521' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:20:29.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:29 smithi184 ceph-mon[27098]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:29.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:29 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/1931326521' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-02-07T12:20:30.996 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:20:30.996 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":59,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","created":"2024-02-07T12:14:19.506549+0000","modified":"2024-02-07T12:20:18.264682+0000","last_up_change":"2024-02-07T12:20:14.240676+0000","last_in_change":"2024-02-07T12:19:57.278587+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-02-07T12:18:14.333876+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"67efbac6-68ee-4b1d-b17b-30ccdecd5fd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6803","nonce":3253424960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6805","nonce":3253424960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6809","nonce":3253424960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6807","nonce":3253424960}]},"public_addr":"172.21.15.174:6803/3253424960","cluster_addr":"172.21.15.174:6805/3253424960","heartbeat_back_addr":"172.21.15.174:6809/3253424960","heartbeat_front_addr":"172.21.15.174:6807/3253424960","state":["exists","up"]},{"osd":1,"uuid":"648a2717-71c4-4f4b-a47b-1ef7562ce576","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6811","nonce":3157081833}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6813","nonce":3157081833}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6817","nonce":3157081833}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6815","nonce":3157081833}]},"public_addr":"172.21.15.174:6811/3157081833","cluster_addr":"172.21.15.174:6813/3157081833","heartbeat_back_addr":"172.21.15.174:6817/3157081833","heartbeat_front_addr":"172.21.15.174:6815/3157081833","state":["exists","up"]},{"osd":2,"uuid":"c005d20a-a54c-4d99-bf9b-18c11f2accf5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6819","nonce":2063842846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6821","nonce":2063842846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6825","nonce":2063842846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6823","nonce":2063842846}]},"public_addr":"172.21.15.174:6819/2063842846","cluster_addr":"172.21.15.174:6821/2063842846","heartbeat_back_addr":"172.21.15.174:6825/2063842846","heartbeat_front_addr":"172.21.15.174:6823/2063842846","state":["exists","up"]},{"osd":3,"uuid":"8b9a6dc7-f3e0-44b7-a60f-f6bd8f230df4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6827","nonce":1389980276}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6829","nonce":1389980276}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6832","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6833","nonce":1389980276}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6831","nonce":1389980276}]},"public_addr":"172.21.15.174:6827/1389980276","cluster_addr":"172.21.15.174:6829/1389980276","heartbeat_back_addr":"172.21.15.174:6833/1389980276","heartbeat_front_addr":"172.21.15.174:6831/1389980276","state":["exists","up"]},{"osd":4,"uuid":"f6700226-b7c0-48c6-b600-c3d14046351d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6800","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6801","nonce":764775003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6802","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6803","nonce":764775003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6806","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6807","nonce":764775003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6804","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6805","nonce":764775003}]},"public_addr":"172.21.15.184:6801/764775003","cluster_addr":"172.21.15.184:6803/764775003","heartbeat_back_addr":"172.21.15.184:6807/764775003","heartbeat_front_addr":"172.21.15.184:6805/764775003","state":["exists","up"]},{"osd":5,"uuid":"0732d783-5be3-4429-8a53-b575b0d5f880","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6808","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6809","nonce":559985402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6810","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6811","nonce":559985402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6814","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6815","nonce":559985402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6812","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6813","nonce":559985402}]},"public_addr":"172.21.15.184:6809/559985402","cluster_addr":"172.21.15.184:6811/559985402","heartbeat_back_addr":"172.21.15.184:6815/559985402","heartbeat_front_addr":"172.21.15.184:6813/559985402","state":["exists","up"]},{"osd":6,"uuid":"d8aa24f1-32f7-46ea-b44e-490ffeeacbd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6816","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6817","nonce":273946647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6818","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6819","nonce":273946647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6822","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6823","nonce":273946647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6820","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6821","nonce":273946647}]},"public_addr":"172.21.15.184:6817/273946647","cluster_addr":"172.21.15.184:6819/273946647","heartbeat_back_addr":"172.21.15.184:6823/273946647","heartbeat_front_addr":"172.21.15.184:6821/273946647","state":["exists","up"]},{"osd":7,"uuid":"7d161396-67be-4db2-bc1c-88d810d907b5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6824","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6825","nonce":649846180}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6826","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6827","nonce":649846180}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6830","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6831","nonce":649846180}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6828","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6829","nonce":649846180}]},"public_addr":"172.21.15.184:6825/649846180","cluster_addr":"172.21.15.184:6827/649846180","heartbeat_back_addr":"172.21.15.184:6831/649846180","heartbeat_front_addr":"172.21.15.184:6829/649846180","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:17:19.350238+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:17:44.721918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:18:12.247857+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:18:41.409447+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:02.678429+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:25.538675+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:49.235242+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:12.402006+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.174:0/2512161230":"2024-02-08T12:16:11.043144+0000","172.21.15.174:0/4031027938":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6801/3745610189":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6800/3745610189":"2024-02-08T12:16:11.043144+0000","172.21.15.174:0/547150068":"2024-02-08T12:15:13.880259+0000","172.21.15.174:6800/2133101063":"2024-02-08T12:14:44.108968+0000","172.21.15.174:6801/3584666552":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1253517459":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1469259571":"2024-02-08T12:14:44.108968+0000","172.21.15.174:6801/2133101063":"2024-02-08T12:14:44.108968+0000","172.21.15.174:0/1798658282":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6800/3584666552":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/601484952":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1506488865":"2024-02-08T12:14:44.108968+0000","172.21.15.174:0/2510143664":"2024-02-08T12:14:44.108968+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-07T12:20:31.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:31 smithi174 ceph-mon[24580]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:31.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:31 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/559346287' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:20:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:31 smithi184 ceph-mon[27098]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:31 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/559346287' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:20:31.745 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-02-07T12:20:31.745 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd dump --format=json 2024-02-07T12:20:31.958 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:33.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:33 smithi174 ceph-mon[24580]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:33 smithi184 ceph-mon[27098]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:33.621 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:20:33.622 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":59,"fsid":"53d92d24-c5b2-11ee-95b6-87774f69a715","created":"2024-02-07T12:14:19.506549+0000","modified":"2024-02-07T12:20:18.264682+0000","last_up_change":"2024-02-07T12:20:14.240676+0000","last_in_change":"2024-02-07T12:19:57.278587+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-02-07T12:18:14.333876+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"67efbac6-68ee-4b1d-b17b-30ccdecd5fd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6803","nonce":3253424960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6805","nonce":3253424960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6809","nonce":3253424960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":3253424960},{"type":"v1","addr":"172.21.15.174:6807","nonce":3253424960}]},"public_addr":"172.21.15.174:6803/3253424960","cluster_addr":"172.21.15.174:6805/3253424960","heartbeat_back_addr":"172.21.15.174:6809/3253424960","heartbeat_front_addr":"172.21.15.174:6807/3253424960","state":["exists","up"]},{"osd":1,"uuid":"648a2717-71c4-4f4b-a47b-1ef7562ce576","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6811","nonce":3157081833}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6813","nonce":3157081833}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6817","nonce":3157081833}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":3157081833},{"type":"v1","addr":"172.21.15.174:6815","nonce":3157081833}]},"public_addr":"172.21.15.174:6811/3157081833","cluster_addr":"172.21.15.174:6813/3157081833","heartbeat_back_addr":"172.21.15.174:6817/3157081833","heartbeat_front_addr":"172.21.15.174:6815/3157081833","state":["exists","up"]},{"osd":2,"uuid":"c005d20a-a54c-4d99-bf9b-18c11f2accf5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6819","nonce":2063842846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6821","nonce":2063842846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6825","nonce":2063842846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":2063842846},{"type":"v1","addr":"172.21.15.174:6823","nonce":2063842846}]},"public_addr":"172.21.15.174:6819/2063842846","cluster_addr":"172.21.15.174:6821/2063842846","heartbeat_back_addr":"172.21.15.174:6825/2063842846","heartbeat_front_addr":"172.21.15.174:6823/2063842846","state":["exists","up"]},{"osd":3,"uuid":"8b9a6dc7-f3e0-44b7-a60f-f6bd8f230df4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6827","nonce":1389980276}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6829","nonce":1389980276}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6832","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6833","nonce":1389980276}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":1389980276},{"type":"v1","addr":"172.21.15.174:6831","nonce":1389980276}]},"public_addr":"172.21.15.174:6827/1389980276","cluster_addr":"172.21.15.174:6829/1389980276","heartbeat_back_addr":"172.21.15.174:6833/1389980276","heartbeat_front_addr":"172.21.15.174:6831/1389980276","state":["exists","up"]},{"osd":4,"uuid":"f6700226-b7c0-48c6-b600-c3d14046351d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6800","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6801","nonce":764775003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6802","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6803","nonce":764775003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6806","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6807","nonce":764775003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6804","nonce":764775003},{"type":"v1","addr":"172.21.15.184:6805","nonce":764775003}]},"public_addr":"172.21.15.184:6801/764775003","cluster_addr":"172.21.15.184:6803/764775003","heartbeat_back_addr":"172.21.15.184:6807/764775003","heartbeat_front_addr":"172.21.15.184:6805/764775003","state":["exists","up"]},{"osd":5,"uuid":"0732d783-5be3-4429-8a53-b575b0d5f880","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6808","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6809","nonce":559985402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6810","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6811","nonce":559985402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6814","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6815","nonce":559985402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6812","nonce":559985402},{"type":"v1","addr":"172.21.15.184:6813","nonce":559985402}]},"public_addr":"172.21.15.184:6809/559985402","cluster_addr":"172.21.15.184:6811/559985402","heartbeat_back_addr":"172.21.15.184:6815/559985402","heartbeat_front_addr":"172.21.15.184:6813/559985402","state":["exists","up"]},{"osd":6,"uuid":"d8aa24f1-32f7-46ea-b44e-490ffeeacbd7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6816","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6817","nonce":273946647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6818","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6819","nonce":273946647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6822","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6823","nonce":273946647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6820","nonce":273946647},{"type":"v1","addr":"172.21.15.184:6821","nonce":273946647}]},"public_addr":"172.21.15.184:6817/273946647","cluster_addr":"172.21.15.184:6819/273946647","heartbeat_back_addr":"172.21.15.184:6823/273946647","heartbeat_front_addr":"172.21.15.184:6821/273946647","state":["exists","up"]},{"osd":7,"uuid":"7d161396-67be-4db2-bc1c-88d810d907b5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6824","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6825","nonce":649846180}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6826","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6827","nonce":649846180}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6830","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6831","nonce":649846180}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.184:6828","nonce":649846180},{"type":"v1","addr":"172.21.15.184:6829","nonce":649846180}]},"public_addr":"172.21.15.184:6825/649846180","cluster_addr":"172.21.15.184:6827/649846180","heartbeat_back_addr":"172.21.15.184:6831/649846180","heartbeat_front_addr":"172.21.15.184:6829/649846180","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:17:19.350238+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:17:44.721918+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:18:12.247857+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:18:41.409447+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:02.678429+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:25.538675+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:19:49.235242+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-02-07T12:20:12.402006+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.174:0/2512161230":"2024-02-08T12:16:11.043144+0000","172.21.15.174:0/4031027938":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6801/3745610189":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6800/3745610189":"2024-02-08T12:16:11.043144+0000","172.21.15.174:0/547150068":"2024-02-08T12:15:13.880259+0000","172.21.15.174:6800/2133101063":"2024-02-08T12:14:44.108968+0000","172.21.15.174:6801/3584666552":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1253517459":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1469259571":"2024-02-08T12:14:44.108968+0000","172.21.15.174:6801/2133101063":"2024-02-08T12:14:44.108968+0000","172.21.15.174:0/1798658282":"2024-02-08T12:16:11.043144+0000","172.21.15.174:6800/3584666552":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/601484952":"2024-02-08T12:15:13.880259+0000","172.21.15.174:0/1506488865":"2024-02-08T12:14:44.108968+0000","172.21.15.174:0/2510143664":"2024-02-08T12:14:44.108968+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-02-07T12:20:34.260 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:34 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/532540567' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:20:34.284 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-02-07T12:20:34.285 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-02-07T12:20:34.285 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-02-07T12:20:34.286 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-02-07T12:20:34.286 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-02-07T12:20:34.286 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-02-07T12:20:34.287 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-02-07T12:20:34.287 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-02-07T12:20:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:34 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/532540567' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-02-07T12:20:34.696 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:34.703 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:34.703 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:34.703 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:34.709 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:34.710 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:34.712 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:34.736 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:35.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:35 smithi174 ceph-mon[24580]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:35 smithi184 ceph-mon[27098]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:37.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:37 smithi174 ceph-mon[24580]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:37 smithi184 ceph-mon[27098]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:39.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:39 smithi174 ceph-mon[24580]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:39.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:39 smithi184 ceph-mon[27098]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:41.277 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:41 smithi174 ceph-mon[24580]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:41 smithi184 ceph-mon[27098]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:42.955 INFO:teuthology.orchestra.run.smithi174.stdout:167503724562 2024-02-07T12:20:42.955 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-02-07T12:20:43.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:43 smithi174 ceph-mon[24580]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:43 smithi184 ceph-mon[27098]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:45.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:45 smithi174 ceph-mon[24580]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:45 smithi184 ceph-mon[27098]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:46.330 INFO:teuthology.orchestra.run.smithi174.stdout:81604378657 2024-02-07T12:20:46.331 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-02-07T12:20:47.138 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:47.242 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:47 smithi174 ceph-mon[24580]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:47 smithi184 ceph-mon[27098]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:47.540 INFO:teuthology.orchestra.run.smithi174.stdout:111669149724 2024-02-07T12:20:47.541 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-02-07T12:20:48.342 INFO:teuthology.orchestra.run.smithi174.stdout:236223201289 2024-02-07T12:20:48.343 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-02-07T12:20:48.845 INFO:teuthology.orchestra.run.smithi174.stdout:38654705708 2024-02-07T12:20:48.846 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-02-07T12:20:49.056 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:49.056 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:49.354 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:49 smithi174 ceph-mon[24580]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:49.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:49 smithi184 ceph-mon[27098]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:50.586 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:51.276 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:51 smithi174 ceph-mon[24580]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:51 smithi184 ceph-mon[27098]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:51.586 INFO:teuthology.orchestra.run.smithi174.stdout:201863462926 2024-02-07T12:20:51.586 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-02-07T12:20:53.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:53 smithi174 ceph-mon[24580]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:53 smithi184 ceph-mon[27098]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:54.024 INFO:teuthology.orchestra.run.smithi174.stdout:81604378658 2024-02-07T12:20:54.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:54 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/886158572' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-02-07T12:20:54.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:54 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/886158572' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-02-07T12:20:55.147 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:55 smithi174 ceph-mon[24580]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:55.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:55 smithi184 ceph-mon[27098]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:56.225 INFO:teuthology.orchestra.run.smithi174.stdout:60129542183 2024-02-07T12:20:56.226 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-02-07T12:20:56.661 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378658 for osd.2 2024-02-07T12:20:56.662 DEBUG:teuthology.parallel:result is None 2024-02-07T12:20:57.250 INFO:teuthology.orchestra.run.smithi174.stdout:236223201292 2024-02-07T12:20:57.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:57 smithi174 ceph-mon[24580]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:57 smithi184 ceph-mon[27098]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:57.481 INFO:teuthology.orchestra.run.smithi174.stdout:167503724565 2024-02-07T12:20:57.515 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:58 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3290943731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-02-07T12:20:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:58 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2874053970' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-02-07T12:20:58.485 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:58 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3290943731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-02-07T12:20:58.485 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:58 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2874053970' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-02-07T12:20:58.508 INFO:teuthology.orchestra.run.smithi174.stdout:133143986200 2024-02-07T12:20:58.508 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-02-07T12:20:59.094 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724562 got 167503724565 for osd.5 2024-02-07T12:20:59.094 DEBUG:teuthology.parallel:result is None 2024-02-07T12:20:59.332 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:59.332 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:20:59.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:20:59 smithi184 ceph-mon[27098]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:59.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:20:59 smithi174 ceph-mon[24580]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:20:59.505 INFO:teuthology.orchestra.run.smithi174.stdout:38654705711 2024-02-07T12:20:59.904 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201292 for osd.7 2024-02-07T12:20:59.904 DEBUG:teuthology.parallel:result is None 2024-02-07T12:21:00.135 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:00 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/972461216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-02-07T12:21:00.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:00 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/972461216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-02-07T12:21:01.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:01 smithi174 ceph-mon[24580]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:01 smithi184 ceph-mon[27098]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:02.480 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:21:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:03 smithi184 ceph-mon[27098]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:03.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:03 smithi174 ceph-mon[24580]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:03.589 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705711 for osd.0 2024-02-07T12:21:03.589 DEBUG:teuthology.parallel:result is None 2024-02-07T12:21:04.945 INFO:teuthology.orchestra.run.smithi174.stdout:133143986202 2024-02-07T12:21:05.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:05 smithi174 ceph-mon[24580]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:05.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:05 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3938712642' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-02-07T12:21:05.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:05 smithi184 ceph-mon[27098]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:05.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:05 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3938712642' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-02-07T12:21:05.733 INFO:teuthology.orchestra.run.smithi174.stdout:111669149727 2024-02-07T12:21:06.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:06 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2099176814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-02-07T12:21:06.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:06 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2099176814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-02-07T12:21:06.552 INFO:teuthology.orchestra.run.smithi174.stdout:60129542186 2024-02-07T12:21:07.164 INFO:teuthology.orchestra.run.smithi174.stdout:201863462930 2024-02-07T12:21:07.164 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:07 smithi174 ceph-mon[24580]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:07.164 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:07 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2831849825' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-02-07T12:21:07.445 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542186 for osd.1 2024-02-07T12:21:07.445 DEBUG:teuthology.parallel:result is None 2024-02-07T12:21:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:07 smithi184 ceph-mon[27098]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:07 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2831849825' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-02-07T12:21:07.846 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149727 for osd.3 2024-02-07T12:21:07.846 DEBUG:teuthology.parallel:result is None 2024-02-07T12:21:08.257 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986200 got 133143986202 for osd.4 2024-02-07T12:21:08.257 DEBUG:teuthology.parallel:result is None 2024-02-07T12:21:08.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:08 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3182019641' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-02-07T12:21:08.493 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:08 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3182019641' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-02-07T12:21:08.937 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462930 for osd.6 2024-02-07T12:21:08.937 DEBUG:teuthology.parallel:result is None 2024-02-07T12:21:08.937 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-02-07T12:21:08.937 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph pg dump --format=json 2024-02-07T12:21:09.160 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:21:09.282 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:09 smithi174 ceph-mon[24580]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:09.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:09 smithi184 ceph-mon[27098]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:11.028 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:21:11.030 INFO:teuthology.orchestra.run.smithi174.stderr:dumped all 2024-02-07T12:21:11.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:11 smithi174 ceph-mon[24580]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:11 smithi184 ceph-mon[27098]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:11.642 INFO:teuthology.orchestra.run.smithi174.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2024-02-07T12:21:10.064582+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48932,"kb_used_data":6308,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912284,"statfs":{"total":767960285184,"available":767910178816,"internally_reserved":0,"allocated":6459392,"data_stored":3547315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002073"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-02-07T12:20:18.277030+0000","last_change":"2024-02-07T12:20:18.277030+0000","last_active":"2024-02-07T12:20:18.277030+0000","last_peered":"2024-02-07T12:20:18.277030+0000","last_clean":"2024-02-07T12:20:18.277030+0000","last_became_active":"2024-02-07T12:20:18.276578+0000","last_became_peered":"2024-02-07T12:20:18.276578+0000","last_unstale":"2024-02-07T12:20:18.277030+0000","last_undegraded":"2024-02-07T12:20:18.277030+0000","last_fullsized":"2024-02-07T12:20:18.277030+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-02-07T12:18:15.090559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-02-07T12:18:15.090559+0000","last_clean_scrub_stamp":"2024-02-07T12:18:15.090559+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-02-08T16:09:12.903692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53700000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70899999999999996}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65100000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63700000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.745}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73599999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.61699999999999999}]},{"osd":1,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.63400000000000001}]},{"osd":2,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.57199999999999995}]},{"osd":3,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.67900000000000005}]},{"osd":4,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.60399999999999998}]},{"osd":5,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.67200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.32100000000000001},"min":{"1min":0.23000000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.53700000000000003,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.34300000000000003,"15min":0.32500000000000001},"min":{"1min":0.26800000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.372}]},{"osd":2,"last update":"Wed Feb 7 12:20:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.38,"15min":0.373},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.55000000000000004,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.38200000000000001,"15min":0.378},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.54800000000000004,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.39400000000000002}]},{"osd":3,"last update":"Wed Feb 7 12:20:47 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.436,"15min":0.435},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.441,"15min":0.437},"min":{"1min":0.31,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.54100000000000004}]},{"osd":4,"last update":"Wed Feb 7 12:20:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.53100000000000003}]},{"osd":5,"last update":"Wed Feb 7 12:20:30 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.443}]},{"osd":6,"last update":"Wed Feb 7 12:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.48899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48199999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Feb 7 12:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.33500000000000002,"15min":0.317},"min":{"1min":0.23400000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.59399999999999997,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.307},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.34300000000000003,"15min":0.32500000000000001},"min":{"1min":0.222,"5min":0.188,"15min":0.188},"max":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.29399999999999998}]},{"osd":2,"last update":"Wed Feb 7 12:20:21 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.35899999999999999,"15min":0.35199999999999998},"min":{"1min":0.29999999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.378,"15min":0.371},"min":{"1min":0.27700000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.27200000000000002}]},{"osd":3,"last update":"Wed Feb 7 12:20:49 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.56200000000000006,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.42699999999999999,"15min":0.43099999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.33500000000000002}]},{"osd":4,"last update":"Wed Feb 7 12:20:09 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.58399999999999996}]},{"osd":5,"last update":"Wed Feb 7 12:20:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.625}]},{"osd":6,"last update":"Wed Feb 7 12:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.505},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.57599999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65100000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6296,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738856,"statfs":{"total":95995035648,"available":95988588544,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.34399999999999997,"15min":0.33400000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.33300000000000002,"15min":0.32100000000000001},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.312}]},{"osd":1,"last update":"Wed Feb 7 12:20:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.35299999999999998,"15min":0.34399999999999997},"min":{"1min":0.28599999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.35199999999999998,"15min":0.34200000000000003},"min":{"1min":0.27100000000000002,"5min":0.223,"15min":0.223},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.29999999999999999}]},{"osd":3,"last update":"Wed Feb 7 12:20:48 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.40699999999999997,"15min":0.40300000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.34100000000000003}]},{"osd":4,"last update":"Wed Feb 7 12:20:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.624},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.59999999999999998}]},{"osd":5,"last update":"Wed Feb 7 12:20:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.497},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.57799999999999996}]},{"osd":6,"last update":"Wed Feb 7 12:20:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.64000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61499999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.377,"15min":0.374},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54900000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.32600000000000001}]},{"osd":1,"last update":"Wed Feb 7 12:20:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.40500000000000003,"15min":0.40300000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41699999999999998,"15min":0.41499999999999998},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.38}]},{"osd":2,"last update":"Wed Feb 7 12:20:51 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.45200000000000001,"15min":0.45300000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.44}]},{"osd":4,"last update":"Wed Feb 7 12:20:08 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.49199999999999999}]},{"osd":5,"last update":"Wed Feb 7 12:20:29 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.60099999999999998}]},{"osd":6,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.52100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5788,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739364,"statfs":{"total":95995035648,"available":95989108736,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:06 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.433},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.67700000000000005}]},{"osd":1,"last update":"Wed Feb 7 12:20:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.54700000000000004}]},{"osd":2,"last update":"Wed Feb 7 12:20:06 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.501}]},{"osd":3,"last update":"Wed Feb 7 12:20:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56499999999999995}]},{"osd":5,"last update":"Wed Feb 7 12:20:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.60299999999999998}]},{"osd":6,"last update":"Wed Feb 7 12:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.623}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.58899999999999997}]},{"osd":1,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.624}]},{"osd":2,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.64400000000000002}]},{"osd":3,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.66600000000000004}]},{"osd":4,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.48799999999999999}]},{"osd":6,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.53200000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-02-07T12:21:11.644 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph pg dump --format=json 2024-02-07T12:21:11.858 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:21:12.246 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:12 smithi174 ceph-mon[24580]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:12.247 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:21:12.247 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:21:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:12 smithi184 ceph-mon[27098]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:21:12.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:21:13.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:13 smithi174 ceph-mon[24580]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:13 smithi184 ceph-mon[27098]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:13.716 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:21:13.717 INFO:teuthology.orchestra.run.smithi174.stderr:dumped all 2024-02-07T12:21:14.391 INFO:teuthology.orchestra.run.smithi174.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-02-07T12:21:12.064974+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48932,"kb_used_data":6308,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912284,"statfs":{"total":767960285184,"available":767910178816,"internally_reserved":0,"allocated":6459392,"data_stored":3547315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002166"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-02-07T12:20:18.277030+0000","last_change":"2024-02-07T12:20:18.277030+0000","last_active":"2024-02-07T12:20:18.277030+0000","last_peered":"2024-02-07T12:20:18.277030+0000","last_clean":"2024-02-07T12:20:18.277030+0000","last_became_active":"2024-02-07T12:20:18.276578+0000","last_became_peered":"2024-02-07T12:20:18.276578+0000","last_unstale":"2024-02-07T12:20:18.277030+0000","last_undegraded":"2024-02-07T12:20:18.277030+0000","last_fullsized":"2024-02-07T12:20:18.277030+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-02-07T12:18:15.090559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-02-07T12:18:15.090559+0000","last_clean_scrub_stamp":"2024-02-07T12:18:15.090559+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-02-08T16:09:12.903692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.497}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55500000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.45100000000000001}]},{"osd":1,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.47799999999999998}]},{"osd":2,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.51100000000000001}]},{"osd":3,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.52600000000000002}]},{"osd":4,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.63700000000000001}]},{"osd":5,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.59599999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66000000000000003}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.32100000000000001},"min":{"1min":0.23000000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.53700000000000003,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.34300000000000003,"15min":0.32500000000000001},"min":{"1min":0.26800000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.372}]},{"osd":2,"last update":"Wed Feb 7 12:20:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.38,"15min":0.373},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.55000000000000004,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.38200000000000001,"15min":0.378},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.54800000000000004,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.39400000000000002}]},{"osd":3,"last update":"Wed Feb 7 12:20:47 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.436,"15min":0.435},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.441,"15min":0.437},"min":{"1min":0.31,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.54100000000000004}]},{"osd":4,"last update":"Wed Feb 7 12:20:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.53100000000000003}]},{"osd":5,"last update":"Wed Feb 7 12:20:30 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.443}]},{"osd":6,"last update":"Wed Feb 7 12:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.48899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48199999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Feb 7 12:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.33500000000000002,"15min":0.317},"min":{"1min":0.23400000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.59399999999999997,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.34300000000000003,"15min":0.32500000000000001},"min":{"1min":0.222,"5min":0.188,"15min":0.188},"max":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.32000000000000001}]},{"osd":2,"last update":"Wed Feb 7 12:20:21 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.35899999999999999,"15min":0.35199999999999998},"min":{"1min":0.29999999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.378,"15min":0.371},"min":{"1min":0.27700000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.29599999999999999}]},{"osd":3,"last update":"Wed Feb 7 12:20:49 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.56200000000000006,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.378},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.42699999999999999,"15min":0.43099999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.36299999999999999}]},{"osd":4,"last update":"Wed Feb 7 12:20:09 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48799999999999999}]},{"osd":5,"last update":"Wed Feb 7 12:20:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.505}]},{"osd":6,"last update":"Wed Feb 7 12:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.497}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.432}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6296,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738856,"statfs":{"total":95995035648,"available":95988588544,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.34399999999999997,"15min":0.33400000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.33300000000000002,"15min":0.32100000000000001},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.312}]},{"osd":1,"last update":"Wed Feb 7 12:20:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.35299999999999998,"15min":0.34399999999999997},"min":{"1min":0.28599999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.35199999999999998,"15min":0.34200000000000003},"min":{"1min":0.27100000000000002,"5min":0.223,"15min":0.223},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.29999999999999999}]},{"osd":3,"last update":"Wed Feb 7 12:20:48 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.40699999999999997,"15min":0.40300000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.34100000000000003}]},{"osd":4,"last update":"Wed Feb 7 12:20:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.624},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.59999999999999998}]},{"osd":5,"last update":"Wed Feb 7 12:20:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.497},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.57799999999999996}]},{"osd":6,"last update":"Wed Feb 7 12:20:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.64000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61499999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.377,"15min":0.374},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54900000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.32600000000000001}]},{"osd":1,"last update":"Wed Feb 7 12:20:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.40500000000000003,"15min":0.40300000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41699999999999998,"15min":0.41499999999999998},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.38}]},{"osd":2,"last update":"Wed Feb 7 12:20:51 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.45200000000000001,"15min":0.45300000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.44}]},{"osd":4,"last update":"Wed Feb 7 12:20:08 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.49199999999999999}]},{"osd":5,"last update":"Wed Feb 7 12:20:29 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.60099999999999998}]},{"osd":6,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.52100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5788,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739364,"statfs":{"total":95995035648,"available":95989108736,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:06 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.433},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.67700000000000005}]},{"osd":1,"last update":"Wed Feb 7 12:20:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.54700000000000004}]},{"osd":2,"last update":"Wed Feb 7 12:20:06 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.501}]},{"osd":3,"last update":"Wed Feb 7 12:20:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56499999999999995}]},{"osd":5,"last update":"Wed Feb 7 12:20:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.60299999999999998}]},{"osd":6,"last update":"Wed Feb 7 12:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.623}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.58899999999999997}]},{"osd":1,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.624}]},{"osd":2,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.64400000000000002}]},{"osd":3,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.66600000000000004}]},{"osd":4,"last update":"Wed Feb 7 12:20:32 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.48799999999999999}]},{"osd":6,"last update":"Wed Feb 7 12:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.53200000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-02-07T12:21:14.393 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-02-07T12:21:14.393 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-02-07T12:21:14.393 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-02-07T12:21:14.393 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph health --format=json 2024-02-07T12:21:14.612 INFO:teuthology.orchestra.run.smithi174.stderr:Inferring config /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/mon.smithi174/config 2024-02-07T12:21:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:15 smithi184 ceph-mon[27098]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:15 smithi184 ceph-mon[27098]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:15.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:15 smithi174 ceph-mon[24580]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:15.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:15 smithi174 ceph-mon[24580]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:16.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:16.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:16.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:16.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:16 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:16 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:16.488 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:21:16.489 INFO:teuthology.orchestra.run.smithi174.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-02-07T12:21:17.162 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-02-07T12:21:17.162 INFO:tasks.cephadm:Setup complete, yielding 2024-02-07T12:21:17.163 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-07T12:21:17.174 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:21:17.175 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph orch status' 2024-02-07T12:21:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:17 smithi184 ceph-mon[27098]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:17 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3329622812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-02-07T12:21:17.479 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:17 smithi174 ceph-mon[24580]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:17.479 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:17 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3329622812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-02-07T12:21:19.247 INFO:teuthology.orchestra.run.smithi174.stdout:Backend: cephadm 2024-02-07T12:21:19.247 INFO:teuthology.orchestra.run.smithi174.stdout:Available: Yes 2024-02-07T12:21:19.247 INFO:teuthology.orchestra.run.smithi174.stdout:Paused: No 2024-02-07T12:21:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:19 smithi184 ceph-mon[27098]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:19.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:19 smithi174 ceph-mon[24580]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:19.831 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph orch ps' 2024-02-07T12:21:20.384 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:20 smithi174 ceph-mon[24580]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:20 smithi184 ceph-mon[27098]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:21.379 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:21 smithi174 ceph-mon[24580]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:21 smithi184 ceph-mon[27098]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:21.763 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (4m) 2m ago 5m 14.5M - ba2b418f427c 7e9094a54db5 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (5m) 2m ago 5m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (4m) 69s ago 4m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (3m) 2m ago 5m 42.9M - 8.3.5 dad864ee21e9 15ee5e6c5ab7 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e be683bef83c5 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (4m) 69s ago 4m 410M - 17.2.0 e1d6a67b021e 0810966ae8b7 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (6m) 2m ago 7m 44.2M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (4m) 69s ago 4m 39.3M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (5m) 2m ago 5m 15.6M - 1dbe0e931976 56b452262afd 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (4m) 69s ago 4m 17.4M - 1dbe0e931976 7e6cdb451097 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (4m) 2m ago 4m 34.4M 3216M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (3m) 2m ago 3m 34.9M 3216M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:21:21.764 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (3m) 2m ago 3m 34.0M 3216M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:21:21.765 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (2m) 2m ago 2m 28.0M 3216M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:21:21.765 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (2m) 69s ago 2m 36.3M 3984M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:21:21.765 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (2m) 69s ago 2m 35.9M 3984M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:21:21.765 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (97s) 69s ago 96s 33.1M 3984M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:21:21.765 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (73s) 69s ago 73s 10.9M 3984M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:21:21.765 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (4m) 2m ago 4m 54.6M - 514e6a882f6e 90c30949c773 2024-02-07T12:21:22.411 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph orch ls' 2024-02-07T12:21:23.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:23 smithi174 ceph-mon[24580]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:23.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:23 smithi174 ceph-mon[24580]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:23 smithi184 ceph-mon[27098]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:23 smithi184 ceph-mon[27098]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:crash 2/2 2m ago 6m * 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:mgr 2/2 2m ago 6m count:2 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:mon 2/2 2m ago 5m smithi174:172.21.15.174=smithi174;smithi184:172.21.15.184=smithi184;count:2 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:osd 8 2m ago - 2024-02-07T12:21:24.342 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-02-07T12:21:24.999 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph orch host ls' 2024-02-07T12:21:25.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:25 smithi174 ceph-mon[24580]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:25 smithi184 ceph-mon[27098]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:26 smithi184 ceph-mon[27098]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:26.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:26 smithi174 ceph-mon[24580]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:27.041 INFO:teuthology.orchestra.run.smithi174.stdout:HOST ADDR LABELS STATUS 2024-02-07T12:21:27.041 INFO:teuthology.orchestra.run.smithi174.stdout:smithi174 172.21.15.174 2024-02-07T12:21:27.041 INFO:teuthology.orchestra.run.smithi174.stdout:smithi184 172.21.15.184 2024-02-07T12:21:27.041 INFO:teuthology.orchestra.run.smithi174.stdout:2 hosts in cluster 2024-02-07T12:21:27.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:27 smithi174 ceph-mon[24580]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:27 smithi184 ceph-mon[27098]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:27.727 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph orch device ls' 2024-02-07T12:21:28.289 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:28 smithi174 ceph-mon[24580]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:28 smithi184 ceph-mon[27098]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:29.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:29 smithi174 ceph-mon[24580]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:29 smithi184 ceph-mon[27098]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:29.780 INFO:teuthology.orchestra.run.smithi174.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-02-07T12:21:29.781 INFO:teuthology.orchestra.run.smithi174.stdout:smithi174 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62050074400BGN 400G LVM detected, locked 2024-02-07T12:21:29.781 INFO:teuthology.orchestra.run.smithi174.stdout:smithi184 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT626500CE400BGN 400G LVM detected, locked 2024-02-07T12:21:30.442 INFO:teuthology.run_tasks:Running task vip.exec... 2024-02-07T12:21:30.451 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:21:30.451 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-02-07T12:21:30.481 INFO:teuthology.orchestra.run.smithi174.stderr:+ systemctl stop nfs-server 2024-02-07T12:21:30.488 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi184.front.sepia.ceph.com 2024-02-07T12:21:30.488 DEBUG:teuthology.orchestra.run.smithi184:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-02-07T12:21:30.520 INFO:teuthology.orchestra.run.smithi184.stderr:+ systemctl stop nfs-server 2024-02-07T12:21:30.528 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-07T12:21:30.536 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:21:30.536 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph fs volume create foofs' 2024-02-07T12:21:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:31 smithi184 ceph-mon[27098]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:31 smithi184 ceph-mon[27098]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:31.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:31 smithi174 ceph-mon[24580]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:31.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:31 smithi174 ceph-mon[24580]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:33 smithi184 ceph-mon[27098]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:33 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-02-07T12:21:33.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:33 smithi174 ceph-mon[24580]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:33.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:33 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-02-07T12:21:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:34 smithi174 ceph-mon[24580]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:34 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-02-07T12:21:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:34 smithi174 ceph-mon[24580]: osdmap e60: 8 total, 8 up, 8 in 2024-02-07T12:21:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:34 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-02-07T12:21:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:34 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[24576]: 2024-02-07T12:21:34.219+0000 7ff94313f700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:21:34.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:34 smithi184 ceph-mon[27098]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:34.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:34 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-02-07T12:21:34.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:34 smithi184 ceph-mon[27098]: osdmap e60: 8 total, 8 up, 8 in 2024-02-07T12:21:34.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:34 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-02-07T12:21:34.961 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-02-07T12:21:34.974 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-02-07T12:21:34.974 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch ls -f json 2024-02-07T12:21:35.542 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: pgmap v209: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:35.542 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-02-07T12:21:35.542 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: osdmap e61: 8 total, 8 up, 8 in 2024-02-07T12:21:35.542 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-02-07T12:21:35.542 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: osdmap e62: 8 total, 8 up, 8 in 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: fsmap foofs:0 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: Saving service mds.foofs spec with placement count:2 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.vkrzkp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.vkrzkp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-02-07T12:21:35.543 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:35 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: pgmap v209: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: osdmap e61: 8 total, 8 up, 8 in 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: osdmap e62: 8 total, 8 up, 8 in 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: fsmap foofs:0 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: Saving service mds.foofs spec with placement count:2 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.vkrzkp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:21:35.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.vkrzkp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-02-07T12:21:35.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:35 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:36.514 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:36 smithi174 ceph-mon[24580]: Deploying daemon mds.foofs.smithi174.vkrzkp on smithi174 2024-02-07T12:21:36.514 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:36 smithi174 ceph-mon[24580]: osdmap e63: 8 total, 8 up, 8 in 2024-02-07T12:21:36.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:36 smithi184 ceph-mon[27098]: Deploying daemon mds.foofs.smithi174.vkrzkp on smithi174 2024-02-07T12:21:36.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:36 smithi184 ceph-mon[27098]: osdmap e63: 8 total, 8 up, 8 in 2024-02-07T12:21:37.487 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:37 smithi174 ceph-mon[24580]: pgmap v213: 65 pgs: 1 creating+activating, 4 creating+peering, 33 active+clean, 27 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:37.487 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:37 smithi174 ceph-mon[24580]: osdmap e64: 8 total, 8 up, 8 in 2024-02-07T12:21:37.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:37 smithi184 ceph-mon[27098]: pgmap v213: 65 pgs: 1 creating+activating, 4 creating+peering, 33 active+clean, 27 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:37.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:37 smithi184 ceph-mon[27098]: osdmap e64: 8 total, 8 up, 8 in 2024-02-07T12:21:38.163 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:21:38.163 INFO:teuthology.orchestra.run.smithi174.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-02-07T12:15:03.733694Z", "last_refresh": "2024-02-07T12:18:51.497597Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-02-07T12:14:57.633733Z", "last_refresh": "2024-02-07T12:18:51.497663Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-02-07T12:15:00.597245Z", "last_refresh": "2024-02-07T12:18:51.497726Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-02-07T12:21:34.240631Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-02-07T12:21:34.234588Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-02-07T12:14:55.497950Z", "last_refresh": "2024-02-07T12:18:51.497525Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi174:172.21.15.174=smithi174", "smithi184:172.21.15.184=smithi184"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-02-07T12:15:47.354666Z", "last_refresh": "2024-02-07T12:18:51.497381Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-02-07T12:15:01.843096Z", "last_refresh": "2024-02-07T12:18:51.497788Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-02-07T12:18:51.497911Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-02-07T12:14:58.912248Z", "last_refresh": "2024-02-07T12:18:51.497849Z", "ports": [9095], "running": 1, "size": 1}}] 2024-02-07T12:21:39.004 INFO:tasks.cephadm:mds.foofs has 0/2 2024-02-07T12:21:39.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:39 smithi174 ceph-mon[24580]: pgmap v215: 65 pgs: 1 creating+activating, 4 creating+peering, 50 active+clean, 10 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:39.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:39 smithi174 ceph-mon[24580]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:39.664 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:39 smithi184 ceph-mon[27098]: pgmap v215: 65 pgs: 1 creating+activating, 4 creating+peering, 50 active+clean, 10 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:39.664 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:39 smithi184 ceph-mon[27098]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:40.005 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch ls -f json 2024-02-07T12:21:40.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: mds.? [v2:172.21.15.174:6834/1377814839,v1:172.21.15.174:6835/1377814839] up:boot 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: daemon mds.foofs.smithi174.vkrzkp assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: Cluster is now healthy 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: fsmap foofs:0 1 up:standby 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:creating} 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: daemon mds.foofs.smithi174.vkrzkp is now active in filesystem foofs as rank 0 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi184.alpixp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi184.alpixp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:40.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:40 smithi174 ceph-mon[24580]: Deploying daemon mds.foofs.smithi184.alpixp on smithi184 2024-02-07T12:21:40.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: mds.? [v2:172.21.15.174:6834/1377814839,v1:172.21.15.174:6835/1377814839] up:boot 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: daemon mds.foofs.smithi174.vkrzkp assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: Cluster is now healthy 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: fsmap foofs:0 1 up:standby 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:creating} 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: daemon mds.foofs.smithi174.vkrzkp is now active in filesystem foofs as rank 0 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi184.alpixp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi184.alpixp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:40.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:40 smithi184 ceph-mon[27098]: Deploying daemon mds.foofs.smithi184.alpixp on smithi184 2024-02-07T12:21:41.527 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:41 smithi174 ceph-mon[24580]: pgmap v216: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 174 B/s wr, 0 op/s 2024-02-07T12:21:41.527 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:41 smithi174 ceph-mon[24580]: mds.? [v2:172.21.15.174:6834/1377814839,v1:172.21.15.174:6835/1377814839] up:active 2024-02-07T12:21:41.527 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:41 smithi174 ceph-mon[24580]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 2024-02-07T12:21:41.528 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:41 smithi184 ceph-mon[27098]: pgmap v216: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 174 B/s wr, 0 op/s 2024-02-07T12:21:41.528 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:41 smithi184 ceph-mon[27098]: mds.? [v2:172.21.15.174:6834/1377814839,v1:172.21.15.174:6835/1377814839] up:active 2024-02-07T12:21:41.528 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:41 smithi184 ceph-mon[27098]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 2024-02-07T12:21:42.100 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:21:42.100 INFO:teuthology.orchestra.run.smithi174.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-02-07T12:15:03.733694Z", "last_refresh": "2024-02-07T12:18:51.497597Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-02-07T12:14:57.633733Z", "last_refresh": "2024-02-07T12:18:51.497663Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-02-07T12:15:00.597245Z", "last_refresh": "2024-02-07T12:18:51.497726Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-02-07T12:21:34.240631Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-02-07T12:21:34.234588Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-02-07T12:14:55.497950Z", "last_refresh": "2024-02-07T12:18:51.497525Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi174:172.21.15.174=smithi174", "smithi184:172.21.15.184=smithi184"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-02-07T12:15:47.354666Z", "last_refresh": "2024-02-07T12:18:51.497381Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-02-07T12:15:01.843096Z", "last_refresh": "2024-02-07T12:18:51.497788Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-02-07T12:18:51.497911Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-02-07T12:14:58.912248Z", "last_refresh": "2024-02-07T12:18:51.497849Z", "ports": [9095], "running": 1, "size": 1}}] 2024-02-07T12:21:42.771 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 3 op/s 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: mds.? [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] up:boot 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:21:42.772 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:42 smithi184 ceph-mon[27098]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:21:42.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:42.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 3 op/s 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: mds.? [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] up:boot 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:21:42.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:42 smithi174 ceph-mon[24580]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:21:43.679 INFO:tasks.cephadm:mds.foofs has 0/2 2024-02-07T12:21:44.681 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch ls -f json 2024-02-07T12:21:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:44 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:44 smithi184 ceph-mon[27098]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 3 op/s 2024-02-07T12:21:44.706 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:44 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:44.706 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:44 smithi174 ceph-mon[24580]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 3 op/s 2024-02-07T12:21:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:47 smithi184 ceph-mon[27098]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-02-07T12:21:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:47 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:47.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:47 smithi174 ceph-mon[24580]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-02-07T12:21:47.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:47 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:48.378 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:21:48.378 INFO:teuthology.orchestra.run.smithi174.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-02-07T12:15:03.733694Z", "last_refresh": "2024-02-07T12:18:51.497597Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-02-07T12:14:57.633733Z", "last_refresh": "2024-02-07T12:18:51.497663Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-02-07T12:15:00.597245Z", "last_refresh": "2024-02-07T12:18:51.497726Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-02-07T12:21:34.240631Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-02-07T12:21:34.234588Z", "last_refresh": "2024-02-07T12:21:43.413045Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-02-07T12:14:55.497950Z", "last_refresh": "2024-02-07T12:18:51.497525Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi174:172.21.15.174=smithi174", "smithi184:172.21.15.184=smithi184"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-02-07T12:15:47.354666Z", "last_refresh": "2024-02-07T12:18:51.497381Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-02-07T12:15:01.843096Z", "last_refresh": "2024-02-07T12:18:51.497788Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-02-07T12:18:51.497911Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-02-07T12:14:58.912248Z", "last_refresh": "2024-02-07T12:18:51.497849Z", "ports": [9095], "running": 1, "size": 1}}] 2024-02-07T12:21:49.044 INFO:tasks.cephadm:mds.foofs has 1/2 2024-02-07T12:21:49.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:49 smithi174 ceph-mon[24580]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-02-07T12:21:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:49.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:49 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:49.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:49 smithi184 ceph-mon[27098]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-02-07T12:21:49.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:49.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:49.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:49.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:49.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:49.396 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:49 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:50.045 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch ls -f json 2024-02-07T12:21:50.364 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:50 smithi174 ceph-mon[24580]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:50.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:50 smithi184 ceph-mon[27098]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:51.443 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:51 smithi174 ceph-mon[24580]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-02-07T12:21:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:51 smithi184 ceph-mon[27098]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-02-07T12:21:52.175 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:21:52.175 INFO:teuthology.orchestra.run.smithi174.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-02-07T12:15:03.733694Z", "last_refresh": "2024-02-07T12:21:48.386335Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-02-07T12:14:57.633733Z", "last_refresh": "2024-02-07T12:21:43.412405Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-02-07T12:15:00.597245Z", "last_refresh": "2024-02-07T12:21:48.386462Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-02-07T12:21:34.240631Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-02-07T12:21:34.234588Z", "last_refresh": "2024-02-07T12:21:43.413045Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-02-07T12:14:55.497950Z", "last_refresh": "2024-02-07T12:21:43.412554Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi174:172.21.15.174=smithi174", "smithi184:172.21.15.184=smithi184"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-02-07T12:15:47.354666Z", "last_refresh": "2024-02-07T12:21:43.412648Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-02-07T12:15:01.843096Z", "last_refresh": "2024-02-07T12:21:43.412717Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-02-07T12:21:43.412785Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-02-07T12:14:58.912248Z", "last_refresh": "2024-02-07T12:21:48.386586Z", "ports": [9095], "running": 1, "size": 1}}] 2024-02-07T12:21:53.025 INFO:tasks.cephadm:mds.foofs has 2/2 2024-02-07T12:21:53.025 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-07T12:21:53.035 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:21:53.036 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-02-07T12:21:53.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:53 smithi174 ceph-mon[24580]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-02-07T12:21:53.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:53 smithi174 ceph-mon[24580]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:53 smithi184 ceph-mon[27098]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-02-07T12:21:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:53 smithi184 ceph-mon[27098]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:21:55.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:55 smithi174 ceph-mon[24580]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 1 op/s 2024-02-07T12:21:55.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:55 smithi184 ceph-mon[27098]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 1 op/s 2024-02-07T12:21:56.329 INFO:teuthology.orchestra.run.smithi174.stdout:NFS Cluster Created Successfully 2024-02-07T12:21:56.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:56 smithi174 ceph-mon[24580]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:56.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-02-07T12:21:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-02-07T12:21:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:56 smithi174 ceph-mon[24580]: osdmap e65: 8 total, 8 up, 8 in 2024-02-07T12:21:56.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:56 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-02-07T12:21:56.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:56 smithi184 ceph-mon[27098]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:21:56.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-02-07T12:21:56.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-02-07T12:21:56.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:56 smithi184 ceph-mon[27098]: osdmap e65: 8 total, 8 up, 8 in 2024-02-07T12:21:56.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:56 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-02-07T12:21:56.970 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: pgmap v225: 97 pgs: 3 creating+peering, 29 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: osdmap e66: 8 total, 8 up, 8 in 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-02-07T12:21:57.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-07T12:21:57.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-07T12:21:57.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:57.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-07T12:21:57.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-07T12:21:57.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:21:57.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:21:57.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:57 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: pgmap v225: 97 pgs: 3 creating+peering, 29 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: osdmap e66: 8 total, 8 up, 8 in 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-02-07T12:21:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-07T12:21:57.601 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-07T12:21:57.601 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:57.601 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-07T12:21:57.601 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-07T12:21:57.601 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:21:57.601 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:21:57.601 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:57 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:21:58.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:58 smithi174 ceph-mon[24580]: Saving service nfs.foo spec with placement count:2 2024-02-07T12:21:58.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:58 smithi174 ceph-mon[24580]: Creating key for client.nfs.foo.0.0.smithi174.zwcqxx 2024-02-07T12:21:58.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:58 smithi174 ceph-mon[24580]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-07T12:21:58.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:58 smithi174 ceph-mon[24580]: Rados config object exists: conf-nfs.foo 2024-02-07T12:21:58.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:58 smithi174 ceph-mon[24580]: Creating key for client.nfs.foo.0.0.smithi174.zwcqxx-rgw 2024-02-07T12:21:58.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:58 smithi174 ceph-mon[24580]: Deploying daemon nfs.foo.0.0.smithi174.zwcqxx on smithi174 2024-02-07T12:21:58.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:58 smithi174 ceph-mon[24580]: osdmap e67: 8 total, 8 up, 8 in 2024-02-07T12:21:58.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:58 smithi174 ceph-mon[24580]: mgrmap e20: smithi174.whsozo(active, since 5m), standbys: smithi184.prgadw 2024-02-07T12:21:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:58 smithi184 ceph-mon[27098]: Saving service nfs.foo spec with placement count:2 2024-02-07T12:21:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:58 smithi184 ceph-mon[27098]: Creating key for client.nfs.foo.0.0.smithi174.zwcqxx 2024-02-07T12:21:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:58 smithi184 ceph-mon[27098]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-07T12:21:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:58 smithi184 ceph-mon[27098]: Rados config object exists: conf-nfs.foo 2024-02-07T12:21:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:58 smithi184 ceph-mon[27098]: Creating key for client.nfs.foo.0.0.smithi174.zwcqxx-rgw 2024-02-07T12:21:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:58 smithi184 ceph-mon[27098]: Deploying daemon nfs.foo.0.0.smithi174.zwcqxx on smithi174 2024-02-07T12:21:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:58 smithi184 ceph-mon[27098]: osdmap e67: 8 total, 8 up, 8 in 2024-02-07T12:21:58.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:58 smithi184 ceph-mon[27098]: mgrmap e20: smithi174.whsozo(active, since 5m), standbys: smithi184.prgadw 2024-02-07T12:21:59.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:21:59 smithi174 ceph-mon[24580]: pgmap v228: 97 pgs: 3 creating+peering, 19 unknown, 75 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:21:59 smithi184 ceph-mon[27098]: pgmap v228: 97 pgs: 3 creating+peering, 19 unknown, 75 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-02-07T12:21:59.920 INFO:teuthology.orchestra.run.smithi174.stderr:Invalid command: Unexpected argument '--clusterid' 2024-02-07T12:21:59.920 INFO:teuthology.orchestra.run.smithi174.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-02-07T12:21:59.920 INFO:teuthology.orchestra.run.smithi174.stderr:Error EINVAL: invalid command 2024-02-07T12:22:00.279 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:22:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "bind": "/fake", 2024-02-07T12:22:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "fs": "foofs", 2024-02-07T12:22:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "path": "/", 2024-02-07T12:22:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "cluster": "foo", 2024-02-07T12:22:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "mode": "RW" 2024-02-07T12:22:00.280 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:22:01.029 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: pgmap v229: 97 pgs: 3 creating+peering, 94 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: Creating key for client.nfs.foo.1.0.smithi184.rixjkj 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-07T12:22:01.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-02-07T12:22:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:01 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-02-07T12:22:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: pgmap v229: 97 pgs: 3 creating+peering, 94 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-02-07T12:22:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: Creating key for client.nfs.foo.1.0.smithi184.rixjkj 2024-02-07T12:22:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-07T12:22:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-02-07T12:22:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-07T12:22:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-07T12:22:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-07T12:22:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-02-07T12:22:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:01 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-02-07T12:22:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:03 smithi184 ceph-mon[27098]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 150 B/s rd, 452 B/s wr, 0 op/s 2024-02-07T12:22:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:03 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-07T12:22:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:03 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-07T12:22:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:03 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:22:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:03 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:22:03.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:03 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:03.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:03 smithi174 ceph-mon[24580]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 150 B/s rd, 452 B/s wr, 0 op/s 2024-02-07T12:22:03.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:03 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-07T12:22:03.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:03 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-07T12:22:03.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:03 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:22:03.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:03 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-02-07T12:22:03.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:03 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:04.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:04 smithi174 ceph-mon[24580]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:04.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:04 smithi174 ceph-mon[24580]: Rados config object exists: conf-nfs.foo 2024-02-07T12:22:04.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:04 smithi174 ceph-mon[24580]: Creating key for client.nfs.foo.1.0.smithi184.rixjkj-rgw 2024-02-07T12:22:04.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:04 smithi174 ceph-mon[24580]: Deploying daemon nfs.foo.1.0.smithi184.rixjkj on smithi184 2024-02-07T12:22:04.565 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:04 smithi184 ceph-mon[27098]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:04.565 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:04 smithi184 ceph-mon[27098]: Rados config object exists: conf-nfs.foo 2024-02-07T12:22:04.565 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:04 smithi184 ceph-mon[27098]: Creating key for client.nfs.foo.1.0.smithi184.rixjkj-rgw 2024-02-07T12:22:04.565 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:04 smithi184 ceph-mon[27098]: Deploying daemon nfs.foo.1.0.smithi184.rixjkj on smithi184 2024-02-07T12:22:05.285 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:05 smithi184 ceph-mon[27098]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 383 B/s wr, 0 op/s 2024-02-07T12:22:05.508 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:05 smithi174 ceph-mon[24580]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 383 B/s wr, 0 op/s 2024-02-07T12:22:06.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:06 smithi174 ceph-mon[24580]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:06.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:06 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:06.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:06 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:06.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:06 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:06.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:06 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:06.621 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:06 smithi184 ceph-mon[27098]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:06.622 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:06 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:06.622 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:06 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:06.622 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:06 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:06.622 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:06 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:07.460 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:07 smithi174 ceph-mon[24580]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:07.461 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:07 smithi174 ceph-mon[24580]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.9 KiB/s wr, 4 op/s 2024-02-07T12:22:07.461 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:07 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:07.505 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:07 smithi184 ceph-mon[27098]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:07.506 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:07 smithi184 ceph-mon[27098]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.9 KiB/s wr, 4 op/s 2024-02-07T12:22:07.506 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:07 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:08.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:08 smithi174 ceph-mon[24580]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:08.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:08 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:08 smithi184 ceph-mon[27098]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:08 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:09.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:09 smithi174 ceph-mon[24580]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-02-07T12:22:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:09 smithi184 ceph-mon[27098]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-02-07T12:22:10.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:10 smithi174 ceph-mon[24580]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:10.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:10 smithi184 ceph-mon[27098]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:10.934 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo ?:2049 2/2 3s ago 14s count:2 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:11 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-02-07T12:22:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:11.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:11.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:11.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:11 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:11.849 INFO:teuthology.run_tasks:Running task vip.exec... 2024-02-07T12:22:11.858 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:22:11.858 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-02-07T12:22:11.888 INFO:teuthology.orchestra.run.smithi174.stderr:+ mkdir /mnt/foo 2024-02-07T12:22:11.903 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-02-07T12:22:11.970 INFO:teuthology.orchestra.run.smithi174.stderr:++ hostname 2024-02-07T12:22:11.971 INFO:teuthology.orchestra.run.smithi174.stderr:+ mount -t nfs smithi174:/fake /mnt/foo -o sync 2024-02-07T12:22:12.380 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-02-07T12:22:12.447 INFO:teuthology.orchestra.run.smithi174.stderr:+ echo test 2024-02-07T12:22:12.457 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-02-07T12:22:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:12 smithi174 ceph-mon[24580]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-02-07T12:22:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-02-07T12:22:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-02-07T12:22:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-02-07T12:22:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:12.523 INFO:teuthology.orchestra.run.smithi174.stderr:+ sync 2024-02-07T12:22:12.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:12 smithi184 ceph-mon[27098]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:12.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-02-07T12:22:12.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-02-07T12:22:12.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-02-07T12:22:12.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-02-07T12:22:12.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:22:12.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:22:12.709 INFO:teuthology.run_tasks:Running task parallel... 2024-02-07T12:22:12.720 INFO:teuthology.task.parallel:starting parallel... 2024-02-07T12:22:12.721 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-02-07T12:22:12.721 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-02-07T12:22:12.722 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:22:12.722 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-02-07T12:22:12.722 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-02-07T12:22:12.722 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-02-07T12:22:12.731 INFO:teuthology.task.exec:Executing custom commands... 2024-02-07T12:22:12.731 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:22:12.731 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-02-07T12:22:12.859 INFO:teuthology.orchestra.run.smithi174.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-02-07T12:22:12.859 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:22:12.859 INFO:teuthology.orchestra.run.smithi174.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-02-07T12:22:12.860 INFO:teuthology.orchestra.run.smithi174.stdout:3 of 5 processes prepared for launch 0 sec 2024-02-07T12:22:12.960 INFO:teuthology.orchestra.run.smithi174.stdout:5 of 5 processes prepared for launch 0 sec 2024-02-07T12:22:12.960 INFO:teuthology.orchestra.run.smithi174.stdout:releasing clients 2024-02-07T12:22:13.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:13 smithi174 ceph-mon[24580]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-02-07T12:22:13.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-02-07T12:22:13.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-02-07T12:22:13.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-02-07T12:22:13.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-02-07T12:22:13.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:13 smithi174 ceph-mon[24580]: osdmap e68: 8 total, 8 up, 8 in 2024-02-07T12:22:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:13 smithi184 ceph-mon[27098]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-02-07T12:22:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-02-07T12:22:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-02-07T12:22:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-02-07T12:22:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-02-07T12:22:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:13 smithi184 ceph-mon[27098]: osdmap e68: 8 total, 8 up, 8 in 2024-02-07T12:22:13.960 INFO:teuthology.orchestra.run.smithi174.stdout: 5 298 54.82 MB/sec warmup 1 sec latency 38.780 ms 2024-02-07T12:22:14.341 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:14 smithi174 ceph-mon[24580]: osdmap e69: 8 total, 8 up, 8 in 2024-02-07T12:22:14.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:14 smithi184 ceph-mon[27098]: osdmap e69: 8 total, 8 up, 8 in 2024-02-07T12:22:14.964 INFO:teuthology.orchestra.run.smithi174.stdout: 5 618 52.01 MB/sec warmup 2 sec latency 11.736 ms 2024-02-07T12:22:15.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:15 smithi174 ceph-mon[24580]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 3.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 322 KiB/s wr, 9 op/s 2024-02-07T12:22:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:15 smithi184 ceph-mon[27098]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 3.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 322 KiB/s wr, 9 op/s 2024-02-07T12:22:15.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 1466 43.31 MB/sec warmup 3 sec latency 10.512 ms 2024-02-07T12:22:16.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 2625 36.92 MB/sec warmup 4 sec latency 8.956 ms 2024-02-07T12:22:17.216 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-02-07T12:22:17.410 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:17 smithi174 ceph-mon[24580]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 59 MiB data, 92 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 7.5 MiB/s wr, 336 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:17 smithi184 ceph-mon[27098]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 59 MiB data, 92 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 7.5 MiB/s wr, 336 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:17.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 3727 38.98 MB/sec warmup 5 sec latency 14.793 ms 2024-02-07T12:22:18.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 4455 33.94 MB/sec warmup 6 sec latency 47.854 ms 2024-02-07T12:22:19.421 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:19 smithi174 ceph-mon[24580]: pgmap v240: 97 pgs: 97 active+clean; 88 MiB data, 102 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s wr, 433 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:19 smithi184 ceph-mon[27098]: pgmap v240: 97 pgs: 97 active+clean; 88 MiB data, 102 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s wr, 433 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:19.967 INFO:teuthology.orchestra.run.smithi174.stdout: 5 5576 32.58 MB/sec warmup 7 sec latency 13.895 ms 2024-02-07T12:22:20.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 6513 32.63 MB/sec warmup 8 sec latency 19.311 ms 2024-02-07T12:22:21.433 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:21 smithi174 ceph-mon[24580]: pgmap v241: 97 pgs: 97 active+clean; 120 MiB data, 190 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 17 MiB/s wr, 770 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:21.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:21 smithi184 ceph-mon[27098]: pgmap v241: 97 pgs: 97 active+clean; 120 MiB data, 190 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 17 MiB/s wr, 770 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:21.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 7361 32.73 MB/sec warmup 9 sec latency 16.252 ms 2024-02-07T12:22:22.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 8170 30.41 MB/sec warmup 10 sec latency 18.336 ms 2024-02-07T12:22:23.324 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph config set global log_to_journald false --force' 2024-02-07T12:22:23.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:23 smithi174 ceph-mon[24580]: pgmap v242: 97 pgs: 97 active+clean; 134 MiB data, 267 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 18 MiB/s wr, 1.06k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:23 smithi184 ceph-mon[27098]: pgmap v242: 97 pgs: 97 active+clean; 134 MiB data, 267 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 18 MiB/s wr, 1.06k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:23.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 9165 29.52 MB/sec warmup 11 sec latency 27.081 ms 2024-02-07T12:22:24.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 9899 29.15 MB/sec warmup 12 sec latency 109.071 ms 2024-02-07T12:22:25.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:25 smithi174 ceph-mon[24580]: pgmap v243: 97 pgs: 97 active+clean; 136 MiB data, 271 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 17 MiB/s wr, 1.00k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:25.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:25 smithi184 ceph-mon[27098]: pgmap v243: 97 pgs: 97 active+clean; 136 MiB data, 271 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 17 MiB/s wr, 1.00k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:25.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 10900 30.01 MB/sec warmup 13 sec latency 14.928 ms 2024-02-07T12:22:26.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 11590 28.29 MB/sec warmup 14 sec latency 22.272 ms 2024-02-07T12:22:27.413 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:27 smithi174 ceph-mon[24580]: pgmap v244: 97 pgs: 97 active+clean; 146 MiB data, 343 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 19 MiB/s wr, 1.33k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:27 smithi184 ceph-mon[27098]: pgmap v244: 97 pgs: 97 active+clean; 146 MiB data, 343 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 19 MiB/s wr, 1.33k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-02-07T12:22:27.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 12566 27.92 MB/sec warmup 15 sec latency 17.682 ms 2024-02-07T12:22:28.421 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:28 smithi174 ceph-mon[24580]: pgmap v245: 97 pgs: 97 active+clean; 151 MiB data, 397 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 15 MiB/s wr, 1.23k op/s 2024-02-07T12:22:28.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:28 smithi184 ceph-mon[27098]: pgmap v245: 97 pgs: 97 active+clean; 151 MiB data, 397 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 15 MiB/s wr, 1.23k op/s 2024-02-07T12:22:28.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 13558 28.16 MB/sec warmup 16 sec latency 16.048 ms 2024-02-07T12:22:29.810 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph mgr module enable nfs --force' 2024-02-07T12:22:29.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 14441 28.51 MB/sec warmup 17 sec latency 23.460 ms 2024-02-07T12:22:30.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 15013 27.22 MB/sec warmup 18 sec latency 16.120 ms 2024-02-07T12:22:31.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:31 smithi174 ceph-mon[24580]: pgmap v246: 97 pgs: 97 active+clean; 155 MiB data, 419 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 15 MiB/s wr, 1.30k op/s 2024-02-07T12:22:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:31 smithi184 ceph-mon[27098]: pgmap v246: 97 pgs: 97 active+clean; 155 MiB data, 419 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 15 MiB/s wr, 1.30k op/s 2024-02-07T12:22:31.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 15846 27.01 MB/sec warmup 19 sec latency 20.368 ms 2024-02-07T12:22:32.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 16918 26.78 MB/sec warmup 20 sec latency 18.333 ms 2024-02-07T12:22:33.197 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:33 smithi174 ceph-mon[24580]: pgmap v247: 97 pgs: 97 active+clean; 164 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 MiB/s wr, 1.36k op/s 2024-02-07T12:22:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:33 smithi184 ceph-mon[27098]: pgmap v247: 97 pgs: 97 active+clean; 164 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 MiB/s wr, 1.36k op/s 2024-02-07T12:22:33.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 17976 27.59 MB/sec warmup 21 sec latency 17.669 ms 2024-02-07T12:22:34.167 INFO:teuthology.orchestra.run.smithi174.stderr:module 'nfs' is already enabled 2024-02-07T12:22:34.201 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:34 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/184247375' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-02-07T12:22:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:34 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/184247375' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-02-07T12:22:34.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 18636 26.69 MB/sec warmup 22 sec latency 26.775 ms 2024-02-07T12:22:35.207 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:35 smithi174 ceph-mon[24580]: pgmap v248: 97 pgs: 97 active+clean; 168 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-02-07T12:22:35.208 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:35 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/184247375' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-02-07T12:22:35.208 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:35 smithi174 ceph-mon[24580]: mgrmap e21: smithi174.whsozo(active, since 6m), standbys: smithi184.prgadw 2024-02-07T12:22:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:35 smithi184 ceph-mon[27098]: pgmap v248: 97 pgs: 97 active+clean; 168 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-02-07T12:22:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:35 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/184247375' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-02-07T12:22:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:35 smithi184 ceph-mon[27098]: mgrmap e21: smithi174.whsozo(active, since 6m), standbys: smithi184.prgadw 2024-02-07T12:22:35.514 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-02-07T12:22:35.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 19486 26.49 MB/sec warmup 23 sec latency 16.172 ms 2024-02-07T12:22:36.966 INFO:teuthology.orchestra.run.smithi174.stdout: 5 20455 26.25 MB/sec warmup 24 sec latency 20.938 ms 2024-02-07T12:22:37.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:37 smithi174 ceph-mon[24580]: pgmap v249: 97 pgs: 97 active+clean; 172 MiB data, 584 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 14 MiB/s wr, 1.32k op/s 2024-02-07T12:22:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:37 smithi184 ceph-mon[27098]: pgmap v249: 97 pgs: 97 active+clean; 172 MiB data, 584 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 14 MiB/s wr, 1.32k op/s 2024-02-07T12:22:37.967 INFO:teuthology.orchestra.run.smithi174.stdout: 5 21503 26.97 MB/sec warmup 25 sec latency 13.604 ms 2024-02-07T12:22:38.967 INFO:teuthology.orchestra.run.smithi174.stdout: 5 22218 26.29 MB/sec warmup 26 sec latency 16.264 ms 2024-02-07T12:22:39.112 INFO:teuthology.orchestra.run.smithi174.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:22:39.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:39 smithi174 ceph-mon[24580]: pgmap v250: 97 pgs: 97 active+clean; 178 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-02-07T12:22:39.211 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:39 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:39.211 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:39 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:39.211 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:39 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:39.211 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:39 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:39.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:39 smithi184 ceph-mon[27098]: pgmap v250: 97 pgs: 97 active+clean; 178 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-02-07T12:22:39.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:39 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:39.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:39 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:22:39.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:39 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:22:39.394 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:39 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:22:39.967 INFO:teuthology.orchestra.run.smithi174.stdout: 5 22920 25.89 MB/sec warmup 27 sec latency 21.251 ms 2024-02-07T12:22:40.219 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:40 smithi174 ceph-mon[24580]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:40.220 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:40 smithi174 ceph-mon[24580]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:22:40.220 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:40.220 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:40 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:40.359 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-02-07T12:22:40.360 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:22:40.360 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-02-07T12:22:40.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:40 smithi184 ceph-mon[27098]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:40.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:40 smithi184 ceph-mon[27098]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:22:40.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:40.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:40 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:22:40.967 INFO:teuthology.orchestra.run.smithi174.stdout: 5 23601 25.42 MB/sec warmup 28 sec latency 40.334 ms 2024-02-07T12:22:41.226 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:41 smithi174 ceph-mon[24580]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:22:41.226 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:41 smithi174 ceph-mon[24580]: pgmap v251: 97 pgs: 97 active+clean; 187 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 13 MiB/s wr, 1.16k op/s 2024-02-07T12:22:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:41 smithi184 ceph-mon[27098]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 2024-02-07T12:22:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:41 smithi184 ceph-mon[27098]: pgmap v251: 97 pgs: 97 active+clean; 187 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 13 MiB/s wr, 1.16k op/s 2024-02-07T12:22:41.967 INFO:teuthology.orchestra.run.smithi174.stdout: 5 24238 25.33 MB/sec warmup 29 sec latency 23.920 ms 2024-02-07T12:22:42.972 INFO:teuthology.orchestra.run.smithi174.stdout: 5 25014 25.57 MB/sec warmup 30 sec latency 35.056 ms 2024-02-07T12:22:43.242 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:43 smithi174 ceph-mon[24580]: pgmap v252: 97 pgs: 97 active+clean; 184 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.09k op/s 2024-02-07T12:22:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:43 smithi184 ceph-mon[27098]: pgmap v252: 97 pgs: 97 active+clean; 184 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.09k op/s 2024-02-07T12:22:43.972 INFO:teuthology.orchestra.run.smithi174.stdout: 5 25453 24.99 MB/sec warmup 31 sec latency 28.939 ms 2024-02-07T12:22:44.645 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:22:44.973 INFO:teuthology.orchestra.run.smithi174.stdout: 5 25967 24.51 MB/sec warmup 32 sec latency 30.941 ms 2024-02-07T12:22:45.264 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:45 smithi174 ceph-mon[24580]: pgmap v253: 97 pgs: 97 active+clean; 187 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 880 op/s 2024-02-07T12:22:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:45 smithi184 ceph-mon[27098]: pgmap v253: 97 pgs: 97 active+clean; 187 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 880 op/s 2024-02-07T12:22:45.620 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:22:45.620 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (5m) 35s ago 7m 24.2M - ba2b418f427c 7e9094a54db5 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (7m) 35s ago 7m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (6m) 37s ago 6m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (5m) 35s ago 6m 52.7M - 8.3.5 dad864ee21e9 15ee5e6c5ab7 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (67s) 35s ago 66s 13.5M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (64s) 37s ago 64s 11.0M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:9283 running (8m) 35s ago 8m 460M - 17.2.0 e1d6a67b021e be683bef83c5 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (6m) 37s ago 6m 411M - 17.2.0 e1d6a67b021e 0810966ae8b7 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (8m) 35s ago 8m 45.9M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (6m) 37s ago 6m 37.0M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (45s) 35s ago 45s 49.9M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (40s) 37s ago 40s 46.6M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 35s ago 6m 15.9M - 1dbe0e931976 56b452262afd 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (6m) 37s ago 6m 17.3M - 1dbe0e931976 7e6cdb451097 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (5m) 35s ago 5m 43.9M 3216M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (5m) 35s ago 5m 42.8M 3216M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:22:45.621 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (4m) 35s ago 4m 42.0M 3216M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:22:45.622 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (4m) 35s ago 4m 42.7M 3216M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:22:45.622 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (3m) 37s ago 3m 42.7M 3984M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:22:45.622 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (3m) 37s ago 3m 43.5M 3984M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:22:45.622 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (3m) 37s ago 3m 41.6M 3984M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:22:45.622 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (2m) 37s ago 2m 41.2M 3984M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:22:45.622 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (5m) 35s ago 5m 63.3M - 514e6a882f6e 90c30949c773 2024-02-07T12:22:45.973 INFO:teuthology.orchestra.run.smithi174.stdout: 5 26743 24.28 MB/sec warmup 33 sec latency 32.561 ms 2024-02-07T12:22:46.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:46 smithi174 ceph-mon[24580]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:46 smithi174 ceph-mon[24580]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:46.279 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:22:46.280 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:22:46.281 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:22:46.281 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:22:46.281 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:22:46.281 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-02-07T12:22:46.281 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:22:46.281 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:22:46.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:46 smithi184 ceph-mon[27098]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:46.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:46 smithi184 ceph-mon[27098]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:46.815 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:22:46.815 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", 2024-02-07T12:22:46.815 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:22:46.816 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [], 2024-02-07T12:22:46.816 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "", 2024-02-07T12:22:46.816 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 image" 2024-02-07T12:22:46.816 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:22:46.973 INFO:teuthology.orchestra.run.smithi174.stdout: 5 27476 24.10 MB/sec warmup 34 sec latency 35.736 ms 2024-02-07T12:22:47.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:47 smithi174 ceph-mon[24580]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:47.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:47 smithi174 ceph-mon[24580]: pgmap v254: 97 pgs: 97 active+clean; 199 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 12 MiB/s wr, 1.01k op/s 2024-02-07T12:22:47.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:47 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3617894517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:22:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:47 smithi184 ceph-mon[27098]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:47 smithi184 ceph-mon[27098]: pgmap v254: 97 pgs: 97 active+clean; 199 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 12 MiB/s wr, 1.01k op/s 2024-02-07T12:22:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:47 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3617894517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:22:47.462 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:22:47.973 INFO:teuthology.orchestra.run.smithi174.stdout: 5 28175 24.25 MB/sec warmup 35 sec latency 28.350 ms 2024-02-07T12:22:48.284 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:48 smithi174 ceph-mon[24580]: from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:48.284 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:48 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/267869639' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:22:48.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:48 smithi184 ceph-mon[27098]: from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:22:48.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:48 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/267869639' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:22:48.973 INFO:teuthology.orchestra.run.smithi174.stdout: 5 28692 24.07 MB/sec warmup 36 sec latency 22.982 ms 2024-02-07T12:22:49.295 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:49 smithi174 ceph-mon[24580]: pgmap v255: 97 pgs: 97 active+clean; 201 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 908 op/s 2024-02-07T12:22:49.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:49 smithi184 ceph-mon[27098]: pgmap v255: 97 pgs: 97 active+clean; 201 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 908 op/s 2024-02-07T12:22:49.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 29220 23.60 MB/sec warmup 37 sec latency 28.379 ms 2024-02-07T12:22:50.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 29842 23.40 MB/sec warmup 38 sec latency 24.082 ms 2024-02-07T12:22:51.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:51 smithi174 ceph-mon[24580]: pgmap v256: 97 pgs: 97 active+clean; 203 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 926 op/s 2024-02-07T12:22:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:51 smithi184 ceph-mon[27098]: pgmap v256: 97 pgs: 97 active+clean; 203 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 926 op/s 2024-02-07T12:22:51.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 30665 23.15 MB/sec warmup 39 sec latency 25.977 ms 2024-02-07T12:22:52.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 31358 23.15 MB/sec warmup 40 sec latency 23.657 ms 2024-02-07T12:22:53.338 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:53 smithi174 ceph-mon[24580]: pgmap v257: 97 pgs: 97 active+clean; 207 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 832 op/s 2024-02-07T12:22:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:53 smithi184 ceph-mon[27098]: pgmap v257: 97 pgs: 97 active+clean; 207 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 832 op/s 2024-02-07T12:22:53.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 32156 23.44 MB/sec warmup 41 sec latency 25.018 ms 2024-02-07T12:22:54.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 32720 23.06 MB/sec warmup 42 sec latency 37.362 ms 2024-02-07T12:22:55.237 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:55 smithi174 ceph-mon[24580]: pgmap v258: 97 pgs: 97 active+clean; 211 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 836 op/s 2024-02-07T12:22:55.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:55 smithi184 ceph-mon[27098]: pgmap v258: 97 pgs: 97 active+clean; 211 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 836 op/s 2024-02-07T12:22:55.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 33391 22.90 MB/sec warmup 43 sec latency 26.763 ms 2024-02-07T12:22:56.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 34279 22.70 MB/sec warmup 44 sec latency 14.775 ms 2024-02-07T12:22:57.247 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:57 smithi174 ceph-mon[24580]: pgmap v259: 97 pgs: 97 active+clean; 217 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-02-07T12:22:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:57 smithi184 ceph-mon[27098]: pgmap v259: 97 pgs: 97 active+clean; 217 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-02-07T12:22:57.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 35197 23.01 MB/sec warmup 45 sec latency 21.906 ms 2024-02-07T12:22:58.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 35969 23.05 MB/sec warmup 46 sec latency 18.000 ms 2024-02-07T12:22:59.256 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:22:59 smithi174 ceph-mon[24580]: pgmap v260: 97 pgs: 97 active+clean; 216 MiB data, 940 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 766 op/s 2024-02-07T12:22:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:22:59 smithi184 ceph-mon[27098]: pgmap v260: 97 pgs: 97 active+clean; 216 MiB data, 940 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 766 op/s 2024-02-07T12:22:59.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 36676 22.80 MB/sec warmup 47 sec latency 23.744 ms 2024-02-07T12:23:00.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 37533 22.72 MB/sec warmup 48 sec latency 33.778 ms 2024-02-07T12:23:01.265 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:01 smithi174 ceph-mon[24580]: pgmap v261: 97 pgs: 97 active+clean; 227 MiB data, 1014 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 944 op/s 2024-02-07T12:23:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:01 smithi184 ceph-mon[27098]: pgmap v261: 97 pgs: 97 active+clean; 227 MiB data, 1014 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 944 op/s 2024-02-07T12:23:01.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 38283 22.71 MB/sec warmup 49 sec latency 27.614 ms 2024-02-07T12:23:02.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 39065 22.90 MB/sec warmup 50 sec latency 22.500 ms 2024-02-07T12:23:03.282 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:03 smithi174 ceph-mon[24580]: pgmap v262: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-02-07T12:23:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:03 smithi184 ceph-mon[27098]: pgmap v262: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-02-07T12:23:03.974 INFO:teuthology.orchestra.run.smithi174.stdout: 5 39596 22.67 MB/sec warmup 51 sec latency 24.810 ms 2024-02-07T12:23:04.975 INFO:teuthology.orchestra.run.smithi174.stdout: 5 40127 22.43 MB/sec warmup 52 sec latency 19.165 ms 2024-02-07T12:23:05.291 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:05 smithi174 ceph-mon[24580]: pgmap v263: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 923 op/s 2024-02-07T12:23:05.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:05 smithi184 ceph-mon[27098]: pgmap v263: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 923 op/s 2024-02-07T12:23:05.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 40727 22.21 MB/sec warmup 53 sec latency 26.022 ms 2024-02-07T12:23:06.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 41427 22.08 MB/sec warmup 54 sec latency 16.773 ms 2024-02-07T12:23:07.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:07 smithi174 ceph-mon[24580]: pgmap v264: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-02-07T12:23:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:07 smithi184 ceph-mon[27098]: pgmap v264: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-02-07T12:23:07.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 42069 22.09 MB/sec warmup 55 sec latency 28.371 ms 2024-02-07T12:23:08.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 42563 22.09 MB/sec warmup 56 sec latency 43.234 ms 2024-02-07T12:23:09.245 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:09 smithi174 ceph-mon[24580]: pgmap v265: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 943 op/s 2024-02-07T12:23:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:09 smithi184 ceph-mon[27098]: pgmap v265: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 943 op/s 2024-02-07T12:23:09.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 42954 21.94 MB/sec warmup 57 sec latency 27.761 ms 2024-02-07T12:23:10.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 43379 21.64 MB/sec warmup 58 sec latency 19.995 ms 2024-02-07T12:23:11.236 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:11 smithi174 ceph-mon[24580]: pgmap v266: 97 pgs: 97 active+clean; 249 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-02-07T12:23:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:11 smithi184 ceph-mon[27098]: pgmap v266: 97 pgs: 97 active+clean; 249 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-02-07T12:23:11.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 44054 21.54 MB/sec warmup 59 sec latency 25.066 ms 2024-02-07T12:23:12.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:23:12.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:23:12.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:23:12.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:23:12.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 44890 21.43 MB/sec warmup 60 sec latency 20.164 ms 2024-02-07T12:23:13.484 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:13 smithi174 ceph-mon[24580]: pgmap v267: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 861 op/s 2024-02-07T12:23:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:13 smithi184 ceph-mon[27098]: pgmap v267: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 861 op/s 2024-02-07T12:23:13.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 45753 21.61 MB/sec warmup 61 sec latency 19.242 ms 2024-02-07T12:23:14.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 46355 21.63 MB/sec warmup 62 sec latency 98.183 ms 2024-02-07T12:23:15.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:15 smithi174 ceph-mon[24580]: pgmap v268: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 715 op/s 2024-02-07T12:23:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:15 smithi184 ceph-mon[27098]: pgmap v268: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 715 op/s 2024-02-07T12:23:15.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 47045 21.49 MB/sec warmup 63 sec latency 57.903 ms 2024-02-07T12:23:16.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 47854 21.43 MB/sec warmup 64 sec latency 128.619 ms 2024-02-07T12:23:17.489 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:17 smithi174 ceph-mon[24580]: pgmap v269: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 955 op/s 2024-02-07T12:23:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:17 smithi184 ceph-mon[27098]: pgmap v269: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 955 op/s 2024-02-07T12:23:17.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 48619 21.35 MB/sec warmup 65 sec latency 108.656 ms 2024-02-07T12:23:17.981 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:23:18.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 49401 21.53 MB/sec warmup 66 sec latency 74.341 ms 2024-02-07T12:23:19.034 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:23:19.034 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (6m) 68s ago 7m 24.2M - ba2b418f427c 7e9094a54db5 2024-02-07T12:23:19.034 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (7m) 68s ago 7m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:23:19.034 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (6m) 71s ago 6m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:23:19.034 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (5m) 68s ago 7m 52.7M - 8.3.5 dad864ee21e9 15ee5e6c5ab7 2024-02-07T12:23:19.034 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (101s) 68s ago 99s 13.5M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (97s) 71s ago 97s 11.0M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:9283 running (8m) 68s ago 8m 460M - 17.2.0 e1d6a67b021e be683bef83c5 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (6m) 71s ago 6m 411M - 17.2.0 e1d6a67b021e 0810966ae8b7 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (8m) 68s ago 8m 45.9M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (6m) 71s ago 6m 37.0M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (79s) 68s ago 78s 49.9M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (74s) 71s ago 73s 46.6M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 68s ago 7m 15.9M - 1dbe0e931976 56b452262afd 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (6m) 71s ago 6m 17.3M - 1dbe0e931976 7e6cdb451097 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (6m) 68s ago 6m 43.9M 3216M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (5m) 68s ago 5m 42.8M 3216M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (5m) 68s ago 5m 42.0M 3216M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (4m) 68s ago 4m 42.7M 3216M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:23:19.035 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (4m) 71s ago 4m 42.7M 3984M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:23:19.036 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (3m) 71s ago 3m 43.5M 3984M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:23:19.036 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (3m) 71s ago 3m 41.6M 3984M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:23:19.036 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (3m) 71s ago 3m 41.2M 3984M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:23:19.036 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (6m) 68s ago 6m 63.3M - 514e6a882f6e 90c30949c773 2024-02-07T12:23:19.285 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:19 smithi174 ceph-mon[24580]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:19.285 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:19 smithi174 ceph-mon[24580]: pgmap v270: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.5 MiB/s wr, 829 op/s 2024-02-07T12:23:19.646 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:23:19.647 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:23:19.648 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-02-07T12:23:19.648 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:23:19.648 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:23:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:19 smithi184 ceph-mon[27098]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:19 smithi184 ceph-mon[27098]: pgmap v270: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.5 MiB/s wr, 829 op/s 2024-02-07T12:23:19.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 49957 21.47 MB/sec warmup 67 sec latency 21.629 ms 2024-02-07T12:23:20.165 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:23:20.165 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261", 2024-02-07T12:23:20.166 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:23:20.166 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [], 2024-02-07T12:23:20.166 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "", 2024-02-07T12:23:20.166 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:44458ccedb18241cc2e17f10cdab3d1385399261 image" 2024-02-07T12:23:20.166 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:23:20.291 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:20 smithi174 ceph-mon[24580]: from='client.24381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:20.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:20 smithi174 ceph-mon[24580]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:20.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:20 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3573856205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:23:20.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:20 smithi184 ceph-mon[27098]: from='client.24381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:20.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:20 smithi184 ceph-mon[27098]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:20.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:20 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3573856205' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:23:20.756 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:23:20.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 50592 21.32 MB/sec warmup 68 sec latency 76.571 ms 2024-02-07T12:23:21.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:21 smithi174 ceph-mon[24580]: pgmap v271: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 856 op/s 2024-02-07T12:23:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:21 smithi174 ceph-mon[24580]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:21 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2776630795' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:23:21.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:21 smithi184 ceph-mon[27098]: pgmap v271: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 856 op/s 2024-02-07T12:23:21.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:21 smithi184 ceph-mon[27098]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:21.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:21 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2776630795' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:23:21.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 51645 21.29 MB/sec warmup 69 sec latency 11.909 ms 2024-02-07T12:23:22.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 52375 21.32 MB/sec warmup 70 sec latency 25.062 ms 2024-02-07T12:23:23.064 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:23:23.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:22 smithi174 ceph-mon[24580]: Upgrade: Target is version 19.0.0-1252-g44458cce (unknown) 2024-02-07T12:23:23.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:22 smithi174 ceph-mon[24580]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:23:23.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:23:23.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:22 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:23:23.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:22 smithi174 ceph-mon[24580]: Upgrade: Need to upgrade myself (mgr.smithi174.whsozo) 2024-02-07T12:23:23.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:22 smithi174 ceph-mon[24580]: pgmap v272: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 799 op/s 2024-02-07T12:23:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:23:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:22 smithi184 ceph-mon[27098]: Upgrade: Target is version 19.0.0-1252-g44458cce (unknown) 2024-02-07T12:23:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:22 smithi184 ceph-mon[27098]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe'] 2024-02-07T12:23:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:23:23.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:22 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:23:23.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:22 smithi184 ceph-mon[27098]: Upgrade: Need to upgrade myself (mgr.smithi174.whsozo) 2024-02-07T12:23:23.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:22 smithi184 ceph-mon[27098]: pgmap v272: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 799 op/s 2024-02-07T12:23:23.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 53215 21.49 MB/sec warmup 71 sec latency 21.935 ms 2024-02-07T12:23:24.068 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:23 smithi174 ceph-mon[24580]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe on smithi184 2024-02-07T12:23:24.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:23 smithi184 ceph-mon[27098]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe on smithi184 2024-02-07T12:23:24.976 INFO:teuthology.orchestra.run.smithi174.stdout: 5 53869 21.39 MB/sec warmup 72 sec latency 19.849 ms 2024-02-07T12:23:25.080 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:24 smithi174 ceph-mon[24580]: pgmap v273: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 973 op/s 2024-02-07T12:23:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:24 smithi184 ceph-mon[27098]: pgmap v273: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 973 op/s 2024-02-07T12:23:25.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 54524 21.32 MB/sec warmup 73 sec latency 23.142 ms 2024-02-07T12:23:26.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 55334 21.22 MB/sec warmup 74 sec latency 23.613 ms 2024-02-07T12:23:27.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:27 smithi174 ceph-mon[24580]: pgmap v274: 97 pgs: 97 active+clean; 287 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-02-07T12:23:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:27 smithi184 ceph-mon[27098]: pgmap v274: 97 pgs: 97 active+clean; 287 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-02-07T12:23:27.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 56210 21.30 MB/sec warmup 75 sec latency 20.428 ms 2024-02-07T12:23:28.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 56970 21.42 MB/sec warmup 76 sec latency 15.333 ms 2024-02-07T12:23:29.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:29 smithi174 ceph-mon[24580]: pgmap v275: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 939 op/s 2024-02-07T12:23:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:29 smithi184 ceph-mon[27098]: pgmap v275: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 939 op/s 2024-02-07T12:23:29.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 57549 21.26 MB/sec warmup 77 sec latency 22.449 ms 2024-02-07T12:23:30.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 58052 21.18 MB/sec warmup 78 sec latency 28.296 ms 2024-02-07T12:23:31.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:31 smithi174 ceph-mon[24580]: pgmap v276: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 976 op/s 2024-02-07T12:23:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:31 smithi184 ceph-mon[27098]: pgmap v276: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 976 op/s 2024-02-07T12:23:31.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 58590 21.00 MB/sec warmup 79 sec latency 57.485 ms 2024-02-07T12:23:32.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 59410 21.02 MB/sec warmup 80 sec latency 85.032 ms 2024-02-07T12:23:33.343 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:33 smithi174 ceph-mon[24580]: pgmap v277: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 991 op/s 2024-02-07T12:23:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:33 smithi184 ceph-mon[27098]: pgmap v277: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 991 op/s 2024-02-07T12:23:33.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 60229 21.16 MB/sec warmup 81 sec latency 168.037 ms 2024-02-07T12:23:34.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 60685 21.08 MB/sec warmup 82 sec latency 84.264 ms 2024-02-07T12:23:35.381 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:35 smithi174 ceph-mon[24580]: pgmap v278: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 1.01k op/s 2024-02-07T12:23:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:35 smithi184 ceph-mon[27098]: pgmap v278: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 1.01k op/s 2024-02-07T12:23:35.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 61324 20.93 MB/sec warmup 83 sec latency 20.168 ms 2024-02-07T12:23:36.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 62107 20.88 MB/sec warmup 84 sec latency 20.538 ms 2024-02-07T12:23:37.391 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:37 smithi174 ceph-mon[24580]: pgmap v279: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 970 op/s 2024-02-07T12:23:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:37 smithi184 ceph-mon[27098]: pgmap v279: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 970 op/s 2024-02-07T12:23:37.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 62947 20.91 MB/sec warmup 85 sec latency 21.508 ms 2024-02-07T12:23:38.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 63835 21.09 MB/sec warmup 86 sec latency 15.332 ms 2024-02-07T12:23:39.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:39 smithi174 ceph-mon[24580]: pgmap v280: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 876 op/s 2024-02-07T12:23:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:39 smithi184 ceph-mon[27098]: pgmap v280: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 876 op/s 2024-02-07T12:23:39.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 64343 20.98 MB/sec warmup 87 sec latency 55.116 ms 2024-02-07T12:23:40.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 64908 20.84 MB/sec warmup 88 sec latency 23.402 ms 2024-02-07T12:23:41.174 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:41 smithi174 ceph-mon[24580]: pgmap v281: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 919 op/s 2024-02-07T12:23:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:41 smithi184 ceph-mon[27098]: pgmap v281: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 919 op/s 2024-02-07T12:23:41.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 65618 20.78 MB/sec warmup 89 sec latency 24.825 ms 2024-02-07T12:23:42.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 66226 20.67 MB/sec warmup 90 sec latency 24.546 ms 2024-02-07T12:23:43.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:43 smithi174 ceph-mon[24580]: pgmap v282: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 861 op/s 2024-02-07T12:23:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:43 smithi184 ceph-mon[27098]: pgmap v282: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 861 op/s 2024-02-07T12:23:43.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 66908 20.73 MB/sec warmup 91 sec latency 24.220 ms 2024-02-07T12:23:44.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 67462 20.77 MB/sec warmup 92 sec latency 36.495 ms 2024-02-07T12:23:45.199 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:45 smithi174 ceph-mon[24580]: pgmap v283: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 843 op/s 2024-02-07T12:23:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:45 smithi184 ceph-mon[27098]: pgmap v283: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 843 op/s 2024-02-07T12:23:45.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 67878 20.66 MB/sec warmup 93 sec latency 35.875 ms 2024-02-07T12:23:46.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 68346 20.49 MB/sec warmup 94 sec latency 31.853 ms 2024-02-07T12:23:47.224 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:47 smithi174 ceph-mon[24580]: pgmap v284: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 912 op/s 2024-02-07T12:23:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:47 smithi184 ceph-mon[27098]: pgmap v284: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 912 op/s 2024-02-07T12:23:47.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 68909 20.46 MB/sec warmup 95 sec latency 31.891 ms 2024-02-07T12:23:48.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 69603 20.34 MB/sec warmup 96 sec latency 26.448 ms 2024-02-07T12:23:49.235 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:49 smithi174 ceph-mon[24580]: pgmap v285: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 862 op/s 2024-02-07T12:23:49.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:49 smithi184 ceph-mon[27098]: pgmap v285: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 862 op/s 2024-02-07T12:23:49.977 INFO:teuthology.orchestra.run.smithi174.stdout: 5 69991 20.24 MB/sec warmup 97 sec latency 67.386 ms 2024-02-07T12:23:50.985 INFO:teuthology.orchestra.run.smithi174.stdout: 5 70520 20.24 MB/sec warmup 98 sec latency 24.917 ms 2024-02-07T12:23:51.180 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:23:51.266 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:51 smithi174 ceph-mon[24580]: pgmap v286: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 877 op/s 2024-02-07T12:23:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:51 smithi184 ceph-mon[27098]: pgmap v286: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 877 op/s 2024-02-07T12:23:51.987 INFO:teuthology.orchestra.run.smithi174.stdout: 5 71008 20.27 MB/sec warmup 99 sec latency 35.443 ms 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (6m) 101s ago 8m 24.2M - ba2b418f427c 7e9094a54db5 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (8m) 101s ago 8m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (7m) 104s ago 7m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (6m) 101s ago 7m 52.7M - 8.3.5 dad864ee21e9 15ee5e6c5ab7 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (2m) 101s ago 2m 13.5M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (2m) 104s ago 2m 11.0M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:9283 running (9m) 101s ago 9m 460M - 17.2.0 e1d6a67b021e be683bef83c5 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (7m) 104s ago 7m 411M - 17.2.0 e1d6a67b021e 0810966ae8b7 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (9m) 101s ago 9m 45.9M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:23:52.036 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (7m) 104s ago 7m 37.0M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (112s) 101s ago 111s 49.9M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (107s) 104s ago 106s 46.6M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 101s ago 7m 15.9M - 1dbe0e931976 56b452262afd 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (7m) 104s ago 7m 17.3M - 1dbe0e931976 7e6cdb451097 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (6m) 101s ago 6m 43.9M 3216M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (6m) 101s ago 6m 42.8M 3216M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (5m) 101s ago 5m 42.0M 3216M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (5m) 101s ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (4m) 104s ago 4m 42.7M 3984M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (4m) 104s ago 4m 43.5M 3984M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (4m) 104s ago 4m 41.6M 3984M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (3m) 104s ago 3m 41.2M 3984M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:23:52.037 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (6m) 101s ago 6m 63.3M - 514e6a882f6e 90c30949c773 2024-02-07T12:23:52.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:52 smithi174 ceph-mon[24580]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:52.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:52 smithi184 ceph-mon[27098]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:52.630 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:23:52.630 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:23:52.630 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:23:52.630 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:23:52.630 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:23:52.630 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:23:52.631 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:23:52.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 71407 20.16 MB/sec warmup 100 sec latency 35.462 ms 2024-02-07T12:23:53.110 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:23:53.110 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:23:53.110 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:23:53.110 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [], 2024-02-07T12:23:53.111 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "0/23 daemons upgraded", 2024-02-07T12:23:53.111 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe image on host smithi184" 2024-02-07T12:23:53.111 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:23:53.288 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:53 smithi174 ceph-mon[24580]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:53.289 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:53 smithi174 ceph-mon[24580]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:53.289 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:53 smithi174 ceph-mon[24580]: pgmap v287: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 849 op/s 2024-02-07T12:23:53.289 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:53 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/683952939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:23:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:53 smithi184 ceph-mon[27098]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:53 smithi184 ceph-mon[27098]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:53 smithi184 ceph-mon[27098]: pgmap v287: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 849 op/s 2024-02-07T12:23:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:53 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/683952939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:23:53.642 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:23:53.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 71907 20.02 MB/sec warmup 101 sec latency 62.047 ms 2024-02-07T12:23:54.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:54 smithi174 ceph-mon[24580]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:54.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:54 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3662394766' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:23:54.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:54 smithi184 ceph-mon[27098]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:23:54.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:54 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3662394766' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:23:54.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 72464 19.99 MB/sec warmup 102 sec latency 77.780 ms 2024-02-07T12:23:55.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:55 smithi174 ceph-mon[24580]: pgmap v288: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 864 op/s 2024-02-07T12:23:55.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:55 smithi184 ceph-mon[27098]: pgmap v288: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 864 op/s 2024-02-07T12:23:55.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 73263 19.91 MB/sec warmup 103 sec latency 45.104 ms 2024-02-07T12:23:56.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 73953 19.97 MB/sec warmup 104 sec latency 73.911 ms 2024-02-07T12:23:57.139 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:57 smithi174 ceph-mon[24580]: pgmap v289: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 897 op/s 2024-02-07T12:23:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:57 smithi184 ceph-mon[27098]: pgmap v289: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 897 op/s 2024-02-07T12:23:57.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 74712 20.07 MB/sec warmup 105 sec latency 24.211 ms 2024-02-07T12:23:58.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 75216 19.94 MB/sec warmup 106 sec latency 44.183 ms 2024-02-07T12:23:59.157 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:23:59 smithi174 ceph-mon[24580]: pgmap v290: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.8 MiB/s wr, 832 op/s 2024-02-07T12:23:59.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:23:59 smithi184 ceph-mon[27098]: pgmap v290: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.8 MiB/s wr, 832 op/s 2024-02-07T12:23:59.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 75618 19.85 MB/sec warmup 107 sec latency 41.942 ms 2024-02-07T12:24:00.915 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:00 smithi174 ceph-mon[24580]: Upgrade: Updating mgr.smithi184.prgadw 2024-02-07T12:24:00.916 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:00.916 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:24:00.916 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:24:00.916 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:00 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:00.916 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:00 smithi174 ceph-mon[24580]: Deploying daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:24:00.916 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:00 smithi174 ceph-mon[24580]: pgmap v291: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 840 op/s 2024-02-07T12:24:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:00 smithi184 ceph-mon[27098]: Upgrade: Updating mgr.smithi184.prgadw 2024-02-07T12:24:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:24:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:24:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:00 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:00 smithi184 ceph-mon[27098]: Deploying daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:24:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:00 smithi184 ceph-mon[27098]: pgmap v291: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 840 op/s 2024-02-07T12:24:00.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 76177 19.78 MB/sec warmup 108 sec latency 34.066 ms 2024-02-07T12:24:01.988 INFO:teuthology.orchestra.run.smithi174.stdout: 5 76785 19.69 MB/sec warmup 109 sec latency 35.414 ms 2024-02-07T12:24:02.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 77676 19.80 MB/sec warmup 110 sec latency 22.155 ms 2024-02-07T12:24:03.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:03 smithi174 ceph-mon[24580]: pgmap v292: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.4 MiB/s wr, 816 op/s 2024-02-07T12:24:03.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:03 smithi184 ceph-mon[27098]: pgmap v292: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.4 MiB/s wr, 816 op/s 2024-02-07T12:24:03.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 78287 19.84 MB/sec warmup 111 sec latency 31.302 ms 2024-02-07T12:24:04.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 78694 19.72 MB/sec warmup 112 sec latency 47.561 ms 2024-02-07T12:24:05.331 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:05 smithi184 ceph-mon[27098]: pgmap v293: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.9 MiB/s wr, 827 op/s 2024-02-07T12:24:05.331 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:05 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:05.331 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:05 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:05.332 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:05 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:05.332 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:05 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:05.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:05 smithi174 ceph-mon[24580]: pgmap v293: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.9 MiB/s wr, 827 op/s 2024-02-07T12:24:05.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:05 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:05.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:05 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:05.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:05 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:05.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:05 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:05.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 79161 19.64 MB/sec warmup 113 sec latency 33.216 ms 2024-02-07T12:24:06.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 79915 19.58 MB/sec warmup 114 sec latency 27.391 ms 2024-02-07T12:24:07.120 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:07 smithi174 ceph-mon[24580]: pgmap v294: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 890 op/s 2024-02-07T12:24:07.194 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:07 smithi184 ceph-mon[27098]: pgmap v294: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 890 op/s 2024-02-07T12:24:07.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 80717 19.60 MB/sec warmup 115 sec latency 34.306 ms 2024-02-07T12:24:08.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 81638 19.76 MB/sec warmup 116 sec latency 19.302 ms 2024-02-07T12:24:09.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:09 smithi174 ceph-mon[24580]: pgmap v295: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 800 op/s 2024-02-07T12:24:09.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:09 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:09 smithi184 ceph-mon[27098]: pgmap v295: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 800 op/s 2024-02-07T12:24:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:09 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:09.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 82142 19.69 MB/sec warmup 117 sec latency 29.596 ms 2024-02-07T12:24:10.390 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:10 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:10.648 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:10 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:10.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 82629 19.58 MB/sec warmup 118 sec latency 24.769 ms 2024-02-07T12:24:11.412 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:11 smithi174 ceph-mon[24580]: pgmap v296: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 838 op/s 2024-02-07T12:24:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:11 smithi184 ceph-mon[27098]: pgmap v296: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 838 op/s 2024-02-07T12:24:11.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 83110 19.54 MB/sec warmup 119 sec latency 41.933 ms 2024-02-07T12:24:12.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:12.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:12 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:12 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:13.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:13 smithi174 ceph-mon[24580]: pgmap v297: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 864 op/s 2024-02-07T12:24:13.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:13.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:13.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:13.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:13.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:13 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr fail", "who": "smithi174.whsozo"}]: dispatch 2024-02-07T12:24:13.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:13 smithi174 ceph-mon[24580]: osdmap e70: 8 total, 8 up, 8 in 2024-02-07T12:24:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:13 smithi184 ceph-mon[27098]: pgmap v297: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 864 op/s 2024-02-07T12:24:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' 2024-02-07T12:24:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:13 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr fail", "who": "smithi174.whsozo"}]: dispatch 2024-02-07T12:24:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:13 smithi184 ceph-mon[27098]: osdmap e70: 8 total, 8 up, 8 in 2024-02-07T12:24:13.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 84095 12.21 MB/sec execute 1 sec latency 58.677 ms 2024-02-07T12:24:14.452 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: Upgrade: Need to upgrade myself (mgr.smithi174.whsozo) 2024-02-07T12:24:14.452 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: Failing over to other MGR 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: Active manager daemon smithi184.prgadw restarted 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: Activating manager daemon smithi184.prgadw 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: osdmap e71: 8 total, 8 up, 8 in 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "mgr fail", "who": "smithi174.whsozo"}]': finished 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: mgrmap e22: smithi184.prgadw(active, starting, since 0.130842s) 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr metadata", "who": "smithi184.prgadw", "id": "smithi184.prgadw"}]: dispatch 2024-02-07T12:24:14.453 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: Manager daemon smithi184.prgadw is now available 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:24:14.454 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi184.prgadw/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:14.455 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi184.prgadw/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:14.455 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi184.prgadw/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:14.455 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:14 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi184.prgadw/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: Upgrade: Need to upgrade myself (mgr.smithi174.whsozo) 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: Failing over to other MGR 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: Active manager daemon smithi184.prgadw restarted 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: Activating manager daemon smithi184.prgadw 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: osdmap e71: 8 total, 8 up, 8 in 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:24:14.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.14204 172.21.15.174:0/49837562' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "mgr fail", "who": "smithi174.whsozo"}]': finished 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: mgrmap e22: smithi184.prgadw(active, starting, since 0.130842s) 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr metadata", "who": "smithi184.prgadw", "id": "smithi184.prgadw"}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:24:14.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: Manager daemon smithi184.prgadw is now available 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi184.prgadw/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi184.prgadw/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi184.prgadw/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:14.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:14 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi184.prgadw/trash_purge_schedule"}]: dispatch 2024-02-07T12:24:14.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 84762 19.05 MB/sec execute 2 sec latency 26.993 ms 2024-02-07T12:24:15.967 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:15 smithi174 ceph-mon[24580]: Deploying cephadm binary to smithi184 2024-02-07T12:24:15.968 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:15 smithi174 ceph-mon[24580]: Deploying cephadm binary to smithi174 2024-02-07T12:24:15.968 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:15 smithi174 ceph-mon[24580]: mgrmap e23: smithi184.prgadw(active, since 1.13463s) 2024-02-07T12:24:15.968 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:15 smithi174 ceph-mon[24580]: pgmap v3: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-02-07T12:24:15.968 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:15 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:15.968 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:15 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:15.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 85345 20.60 MB/sec execute 3 sec latency 27.654 ms 2024-02-07T12:24:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:15 smithi184 ceph-mon[27098]: Deploying cephadm binary to smithi184 2024-02-07T12:24:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:15 smithi184 ceph-mon[27098]: Deploying cephadm binary to smithi174 2024-02-07T12:24:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:15 smithi184 ceph-mon[27098]: mgrmap e23: smithi184.prgadw(active, since 1.13463s) 2024-02-07T12:24:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:15 smithi184 ceph-mon[27098]: pgmap v3: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-02-07T12:24:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:15 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:15 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:16.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 85726 17.13 MB/sec execute 4 sec latency 38.599 ms 2024-02-07T12:24:17.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:17 smithi174 ceph-mon[24580]: pgmap v4: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-02-07T12:24:17.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:17 smithi174 ceph-mon[24580]: [07/Feb/2024:12:24:15] ENGINE Bus STARTING 2024-02-07T12:24:17.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:17 smithi174 ceph-mon[24580]: [07/Feb/2024:12:24:16] ENGINE Serving on http://172.21.15.184:8765 2024-02-07T12:24:17.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:17 smithi174 ceph-mon[24580]: mgrmap e24: smithi184.prgadw(active, since 2s) 2024-02-07T12:24:17.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:17 smithi174 ceph-mon[24580]: [07/Feb/2024:12:24:16] ENGINE Serving on https://172.21.15.184:7150 2024-02-07T12:24:17.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:17 smithi174 ceph-mon[24580]: [07/Feb/2024:12:24:16] ENGINE Bus STARTED 2024-02-07T12:24:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:17 smithi184 ceph-mon[27098]: pgmap v4: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-02-07T12:24:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:17 smithi184 ceph-mon[27098]: [07/Feb/2024:12:24:15] ENGINE Bus STARTING 2024-02-07T12:24:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:17 smithi184 ceph-mon[27098]: [07/Feb/2024:12:24:16] ENGINE Serving on http://172.21.15.184:8765 2024-02-07T12:24:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:17 smithi184 ceph-mon[27098]: mgrmap e24: smithi184.prgadw(active, since 2s) 2024-02-07T12:24:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:17 smithi184 ceph-mon[27098]: [07/Feb/2024:12:24:16] ENGINE Serving on https://172.21.15.184:7150 2024-02-07T12:24:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:17 smithi184 ceph-mon[27098]: [07/Feb/2024:12:24:16] ENGINE Bus STARTED 2024-02-07T12:24:17.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 86075 14.75 MB/sec execute 5 sec latency 39.957 ms 2024-02-07T12:24:18.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 86703 15.00 MB/sec execute 6 sec latency 38.726 ms 2024-02-07T12:24:19.491 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:19 smithi174 ceph-mon[24580]: pgmap v5: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-02-07T12:24:19.491 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:19 smithi174 ceph-mon[24580]: mgrmap e25: smithi184.prgadw(active, since 4s) 2024-02-07T12:24:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:19 smithi184 ceph-mon[27098]: pgmap v5: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-02-07T12:24:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:19 smithi184 ceph-mon[27098]: mgrmap e25: smithi184.prgadw(active, since 4s) 2024-02-07T12:24:19.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 87377 14.33 MB/sec execute 7 sec latency 25.784 ms 2024-02-07T12:24:20.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 87900 14.81 MB/sec execute 8 sec latency 28.011 ms 2024-02-07T12:24:21.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:21 smithi174 ceph-mon[24580]: pgmap v6: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s wr, 229 op/s 2024-02-07T12:24:21.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:21 smithi184 ceph-mon[27098]: pgmap v6: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s wr, 229 op/s 2024-02-07T12:24:21.989 INFO:teuthology.orchestra.run.smithi174.stdout: 5 88462 15.55 MB/sec execute 9 sec latency 38.495 ms 2024-02-07T12:24:22.990 INFO:teuthology.orchestra.run.smithi174.stdout: 5 88933 15.84 MB/sec execute 10 sec latency 33.865 ms 2024-02-07T12:24:23.287 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:23 smithi174 ceph-mon[24580]: pgmap v7: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.1 MiB/s wr, 557 op/s 2024-02-07T12:24:23.288 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:23 smithi174 ceph-mon[24580]: Standby manager daemon smithi174.whsozo started 2024-02-07T12:24:23.288 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:23 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.174:0/319974662' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.whsozo/crt"}]: dispatch 2024-02-07T12:24:23.288 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:23 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.174:0/319974662' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:24:23.288 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:23 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.174:0/319974662' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.whsozo/key"}]: dispatch 2024-02-07T12:24:23.288 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:23 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.174:0/319974662' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:24:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:23 smithi184 ceph-mon[27098]: pgmap v7: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.1 MiB/s wr, 557 op/s 2024-02-07T12:24:23.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:23 smithi184 ceph-mon[27098]: Standby manager daemon smithi174.whsozo started 2024-02-07T12:24:23.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:23 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.174:0/319974662' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.whsozo/crt"}]: dispatch 2024-02-07T12:24:23.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:23 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.174:0/319974662' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:24:23.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:23 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.174:0/319974662' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.whsozo/key"}]: dispatch 2024-02-07T12:24:23.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:23 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.174:0/319974662' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:24:23.990 INFO:teuthology.orchestra.run.smithi174.stdout: 5 89454 15.07 MB/sec execute 11 sec latency 50.713 ms 2024-02-07T12:24:24.210 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:24:24.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:24 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:24:24.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:24 smithi174 ceph-mon[24580]: mgrmap e26: smithi184.prgadw(active, since 9s), standbys: smithi174.whsozo 2024-02-07T12:24:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:24 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:24:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:24 smithi184 ceph-mon[27098]: mgrmap e26: smithi184.prgadw(active, since 9s), standbys: smithi174.whsozo 2024-02-07T12:24:24.990 INFO:teuthology.orchestra.run.smithi174.stdout: 5 90033 15.14 MB/sec execute 12 sec latency 29.782 ms 2024-02-07T12:24:25.363 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:24:25.363 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (7m) 2m ago 8m 24.2M - ba2b418f427c 7e9094a54db5 2024-02-07T12:24:25.363 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (8m) 2m ago 8m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:24:25.363 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (8m) 16s ago 8m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (7m) 2m ago 8m 52.7M - 8.3.5 dad864ee21e9 15ee5e6c5ab7 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (2m) 2m ago 2m 13.5M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (2m) 16s ago 2m 11.5M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e be683bef83c5 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (21s) 16s ago 7m 48.1M - 19.0.0-1252-g44458cce 79957b372307 775a57400e80 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (9m) 2m ago 10m 45.9M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (7m) 16s ago 7m 38.0M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (2m) 2m ago 2m 49.9M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (2m) 16s ago 2m 73.8M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 2m ago 8m 15.9M - 1dbe0e931976 56b452262afd 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (7m) 16s ago 7m 17.2M - 1dbe0e931976 7e6cdb451097 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (7m) 2m ago 7m 43.9M 3216M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (6m) 2m ago 6m 42.8M 3216M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:24:25.364 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (6m) 2m ago 6m 42.0M 3216M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:24:25.365 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (5m) 2m ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:24:25.365 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (5m) 16s ago 5m 286M 3984M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:24:25.365 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (5m) 16s ago 5m 328M 3984M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:24:25.365 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (4m) 16s ago 4m 269M 3984M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:24:25.365 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (4m) 16s ago 4m 387M 3984M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:24:25.365 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (7m) 2m ago 7m 63.3M - 514e6a882f6e 90c30949c773 2024-02-07T12:24:25.414 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:25 smithi174 ceph-mon[24580]: pgmap v8: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.0 MiB/s wr, 433 op/s 2024-02-07T12:24:25.414 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:25 smithi174 ceph-mon[24580]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:25.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:25 smithi184 ceph-mon[27098]: pgmap v8: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.0 MiB/s wr, 433 op/s 2024-02-07T12:24:25.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:25 smithi184 ceph-mon[27098]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:25.990 INFO:teuthology.orchestra.run.smithi174.stdout: 5 90744 14.64 MB/sec execute 13 sec latency 32.098 ms 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:24:26.055 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:24:26.056 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:24:26.056 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:24:26.056 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:24:26.056 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:24:26.056 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:24:26.056 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:24:26.056 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:24:26.423 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:26 smithi174 ceph-mon[24580]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:26.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:26 smithi174 ceph-mon[24580]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:26.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:26 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2695321808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [], 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "1/23 daemons upgraded", 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "", 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:24:26.626 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:24:26.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:26 smithi184 ceph-mon[27098]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:26.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:26 smithi184 ceph-mon[27098]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:26.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:26 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2695321808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:24:26.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 91212 14.60 MB/sec execute 14 sec latency 35.445 ms 2024-02-07T12:24:27.320 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:24:27.428 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:27 smithi174 ceph-mon[24580]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 7.2 MiB/s wr, 698 op/s 2024-02-07T12:24:27.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:27 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:27.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:27 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:27.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:27 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2564210550' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:24:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:27 smithi184 ceph-mon[27098]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 7.2 MiB/s wr, 698 op/s 2024-02-07T12:24:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:27 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:27 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:27 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2564210550' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:24:27.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 91835 15.27 MB/sec execute 15 sec latency 27.890 ms 2024-02-07T12:24:28.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:28 smithi174 ceph-mon[24580]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:28.442 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:28 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:28.442 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:28 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:28.551 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:28 smithi184 ceph-mon[27098]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:28.551 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:28 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:28.552 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:28 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:28.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 92285 15.57 MB/sec execute 16 sec latency 33.576 ms 2024-02-07T12:24:29.458 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:29 smithi174 ceph-mon[24580]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.6 MiB/s wr, 639 op/s 2024-02-07T12:24:29.458 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:29 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:24:29.573 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:29 smithi184 ceph-mon[27098]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.6 MiB/s wr, 639 op/s 2024-02-07T12:24:29.573 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:29 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:24:29.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 92595 14.97 MB/sec execute 17 sec latency 42.298 ms 2024-02-07T12:24:30.733 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:30 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:30.734 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:30 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:30.734 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:30 smithi174 ceph-mon[24580]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.8 MiB/s wr, 652 op/s 2024-02-07T12:24:30.911 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:30 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:30.911 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:30 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:30.911 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:30 smithi184 ceph-mon[27098]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.8 MiB/s wr, 652 op/s 2024-02-07T12:24:30.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 92936 14.48 MB/sec execute 18 sec latency 41.840 ms 2024-02-07T12:24:31.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 93374 14.24 MB/sec execute 19 sec latency 38.642 ms 2024-02-07T12:24:31.999 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:31.999 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:31.999 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:31.999 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:31.999 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:31.999 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:31.999 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi174 to 1936M 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.000 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.001 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.001 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: Adjusting osd_memory_target on smithi184 to 2704M 2024-02-07T12:24:32.001 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.001 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:32.001 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:32.001 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:24:32.001 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:31 smithi174 ceph-mon[24580]: Updating smithi184:/etc/ceph/ceph.conf 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi174 to 1936M 2024-02-07T12:24:32.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: Adjusting osd_memory_target on smithi184 to 2704M 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:24:32.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:24:32.201 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:24:32.201 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:31 smithi184 ceph-mon[27098]: Updating smithi184:/etc/ceph/ceph.conf 2024-02-07T12:24:32.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 93988 14.12 MB/sec execute 20 sec latency 34.657 ms 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: pgmap v12: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 9.3 MiB/s wr, 845 op/s 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: Updating smithi184:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.506 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.506 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.506 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.506 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.506 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.506 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:33 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:24:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: pgmap v12: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 9.3 MiB/s wr, 845 op/s 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: Updating smithi184:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:33 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:33.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 94559 13.99 MB/sec execute 21 sec latency 34.649 ms 2024-02-07T12:24:34.514 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:34 smithi174 ceph-mon[24580]: Reconfiguring alertmanager.smithi174 (dependencies changed)... 2024-02-07T12:24:34.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:34 smithi174 ceph-mon[24580]: Reconfiguring daemon alertmanager.smithi174 on smithi174 2024-02-07T12:24:34.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:34 smithi184 ceph-mon[27098]: Reconfiguring alertmanager.smithi174 (dependencies changed)... 2024-02-07T12:24:34.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:34 smithi184 ceph-mon[27098]: Reconfiguring daemon alertmanager.smithi174 on smithi174 2024-02-07T12:24:34.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 95296 14.57 MB/sec execute 22 sec latency 23.476 ms 2024-02-07T12:24:35.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:35 smithi174 ceph-mon[24580]: pgmap v13: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.9 MiB/s wr, 616 op/s 2024-02-07T12:24:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:35 smithi184 ceph-mon[27098]: pgmap v13: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.9 MiB/s wr, 616 op/s 2024-02-07T12:24:35.992 INFO:teuthology.orchestra.run.smithi174.stdout: 5 95959 15.09 MB/sec execute 23 sec latency 39.917 ms 2024-02-07T12:24:36.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 96614 14.94 MB/sec execute 24 sec latency 22.408 ms 2024-02-07T12:24:37.542 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:37 smithi174 ceph-mon[24580]: pgmap v14: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 932 op/s 2024-02-07T12:24:37.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:37 smithi184 ceph-mon[27098]: pgmap v14: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 932 op/s 2024-02-07T12:24:37.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 97073 14.79 MB/sec execute 25 sec latency 40.452 ms 2024-02-07T12:24:38.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 97714 14.64 MB/sec execute 26 sec latency 43.485 ms 2024-02-07T12:24:39.446 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:39 smithi174 ceph-mon[24580]: pgmap v15: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 618 op/s 2024-02-07T12:24:39.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:39 smithi184 ceph-mon[27098]: pgmap v15: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 618 op/s 2024-02-07T12:24:39.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 98271 14.64 MB/sec execute 27 sec latency 47.589 ms 2024-02-07T12:24:40.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 98962 15.04 MB/sec execute 28 sec latency 22.997 ms 2024-02-07T12:24:41.573 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:41 smithi174 ceph-mon[24580]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 629 op/s 2024-02-07T12:24:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:41 smithi184 ceph-mon[27098]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 629 op/s 2024-02-07T12:24:41.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 99555 15.34 MB/sec execute 29 sec latency 30.224 ms 2024-02-07T12:24:42.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 100109 15.12 MB/sec execute 30 sec latency 20.102 ms 2024-02-07T12:24:43.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:43 smithi174 ceph-mon[24580]: pgmap v17: 97 pgs: 97 active+clean; 316 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 965 op/s 2024-02-07T12:24:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:43 smithi184 ceph-mon[27098]: pgmap v17: 97 pgs: 97 active+clean; 316 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 965 op/s 2024-02-07T12:24:43.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 100589 15.01 MB/sec execute 31 sec latency 28.948 ms 2024-02-07T12:24:44.642 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:44 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:24:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:44 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:24:44.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 101172 14.87 MB/sec execute 32 sec latency 37.722 ms 2024-02-07T12:24:45.647 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:45 smithi174 ceph-mon[24580]: pgmap v18: 97 pgs: 97 active+clean; 316 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 676 op/s 2024-02-07T12:24:45.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:45 smithi184 ceph-mon[27098]: pgmap v18: 97 pgs: 97 active+clean; 316 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 676 op/s 2024-02-07T12:24:45.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 101695 14.80 MB/sec execute 33 sec latency 42.335 ms 2024-02-07T12:24:46.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 102244 14.95 MB/sec execute 34 sec latency 28.131 ms 2024-02-07T12:24:47.657 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:47 smithi174 ceph-mon[24580]: pgmap v19: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-02-07T12:24:47.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:47 smithi184 ceph-mon[27098]: pgmap v19: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-02-07T12:24:47.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 102689 14.97 MB/sec execute 35 sec latency 27.906 ms 2024-02-07T12:24:48.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 103296 15.13 MB/sec execute 36 sec latency 30.827 ms 2024-02-07T12:24:49.685 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:49 smithi174 ceph-mon[24580]: pgmap v20: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 697 op/s 2024-02-07T12:24:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:49 smithi184 ceph-mon[27098]: pgmap v20: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 697 op/s 2024-02-07T12:24:49.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 103669 14.85 MB/sec execute 37 sec latency 40.717 ms 2024-02-07T12:24:50.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 103978 14.64 MB/sec execute 38 sec latency 45.947 ms 2024-02-07T12:24:51.495 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:51 smithi174 ceph-mon[24580]: pgmap v21: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 716 op/s 2024-02-07T12:24:51.496 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:51 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:51.496 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:51 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:51.496 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:51 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:51.496 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:51 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:51.496 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:51 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:24:51.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:51 smithi184 ceph-mon[27098]: pgmap v21: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 716 op/s 2024-02-07T12:24:51.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:51 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:51.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:51 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:51.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:51 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:51.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:51 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:24:51.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:51 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:24:51.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 104316 14.48 MB/sec execute 39 sec latency 49.076 ms 2024-02-07T12:24:52.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:52 smithi174 ceph-mon[24580]: Reconfiguring grafana.smithi174 (dependencies changed)... 2024-02-07T12:24:52.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:52 smithi174 ceph-mon[24580]: Regenerating cephadm self-signed grafana TLS certificates 2024-02-07T12:24:52.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:52 smithi174 ceph-mon[24580]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:24:52.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:52 smithi174 ceph-mon[24580]: Reconfiguring daemon grafana.smithi174 on smithi174 2024-02-07T12:24:52.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:52 smithi184 ceph-mon[27098]: Reconfiguring grafana.smithi174 (dependencies changed)... 2024-02-07T12:24:52.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:52 smithi184 ceph-mon[27098]: Regenerating cephadm self-signed grafana TLS certificates 2024-02-07T12:24:52.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:52 smithi184 ceph-mon[27098]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-02-07T12:24:52.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:52 smithi184 ceph-mon[27098]: Reconfiguring daemon grafana.smithi174 on smithi174 2024-02-07T12:24:52.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 104796 14.30 MB/sec execute 40 sec latency 50.853 ms 2024-02-07T12:24:53.572 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:53 smithi174 ceph-mon[24580]: pgmap v22: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 982 op/s 2024-02-07T12:24:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:53 smithi184 ceph-mon[27098]: pgmap v22: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 982 op/s 2024-02-07T12:24:53.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 105251 14.24 MB/sec execute 41 sec latency 34.414 ms 2024-02-07T12:24:54.606 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:54 smithi174 ceph-mon[24580]: pgmap v23: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.0 MiB/s wr, 634 op/s 2024-02-07T12:24:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:54 smithi184 ceph-mon[27098]: pgmap v23: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.0 MiB/s wr, 634 op/s 2024-02-07T12:24:54.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 105609 14.17 MB/sec execute 42 sec latency 95.005 ms 2024-02-07T12:24:55.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 106056 14.26 MB/sec execute 43 sec latency 45.379 ms 2024-02-07T12:24:56.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 106508 14.38 MB/sec execute 44 sec latency 34.135 ms 2024-02-07T12:24:57.147 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:57 smithi174 ceph-mon[24580]: pgmap v24: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 868 op/s 2024-02-07T12:24:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:57 smithi184 ceph-mon[27098]: pgmap v24: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 868 op/s 2024-02-07T12:24:57.945 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:24:57.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 106836 14.23 MB/sec execute 45 sec latency 35.644 ms 2024-02-07T12:24:58.993 INFO:teuthology.orchestra.run.smithi174.stdout: 5 107112 14.00 MB/sec execute 46 sec latency 95.408 ms 2024-02-07T12:24:59.029 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:24:59.029 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 starting - - - - 2024-02-07T12:24:59.029 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (9m) 29s ago 9m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:24:59.030 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (8m) 32s ago 8m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:24:59.030 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (7m) 29s ago 8m 54.5M - 8.3.5 dad864ee21e9 15ee5e6c5ab7 2024-02-07T12:24:59.030 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (3m) 29s ago 3m 36.9M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:24:59.030 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (3m) 32s ago 3m 11.7M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:24:59.030 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:9283 running (10m) 29s ago 10m 325M - 17.2.0 e1d6a67b021e be683bef83c5 2024-02-07T12:24:59.030 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (54s) 32s ago 8m 487M - 19.0.0-1252-g44458cce 79957b372307 775a57400e80 2024-02-07T12:24:59.031 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (10m) 29s ago 10m 48.2M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:24:59.031 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (8m) 32s ago 8m 38.5M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:24:59.031 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (2m) 29s ago 2m 253M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:24:59.031 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (2m) 32s ago 2m 74.0M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:24:59.031 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 29s ago 8m 18.6M - 1.3.1 1dbe0e931976 56b452262afd 2024-02-07T12:24:59.031 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (8m) 32s ago 8m 17.3M - 1.3.1 1dbe0e931976 7e6cdb451097 2024-02-07T12:24:59.032 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (7m) 29s ago 7m 341M 1936M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:24:59.032 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (7m) 29s ago 7m 262M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:24:59.032 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (6m) 29s ago 6m 206M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:24:59.032 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (6m) 29s ago 6m 295M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:24:59.032 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (6m) 32s ago 6m 329M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:24:59.032 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (5m) 32s ago 5m 346M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:24:59.033 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (5m) 32s ago 5m 273M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:24:59.033 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (4m) 32s ago 4m 435M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:24:59.033 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (8m) 29s ago 8m 65.6M - 2.33.4 514e6a882f6e 90c30949c773 2024-02-07T12:24:59.166 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:59 smithi174 ceph-mon[24580]: pgmap v25: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 531 op/s 2024-02-07T12:24:59.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:59 smithi174 ceph-mon[24580]: from='client.24471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:59.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:24:59 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:24:59.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:59 smithi184 ceph-mon[27098]: pgmap v25: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 531 op/s 2024-02-07T12:24:59.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:59 smithi184 ceph-mon[27098]: from='client.24471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:24:59.395 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:24:59 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:24:59.698 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:24:59.699 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:24:59.700 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:24:59.700 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:24:59.700 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:24:59.700 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:24:59.700 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:24:59.700 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:24:59.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 107397 13.84 MB/sec execute 47 sec latency 44.532 ms 2024-02-07T12:25:00.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:00 smithi174 ceph-mon[24580]: from='client.24475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:00.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:00 smithi174 ceph-mon[24580]: from='client.24479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:00.187 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:00 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/4119637621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:25:00.279 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:25:00.279 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:25:00.279 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:25:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:25:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [], 2024-02-07T12:25:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "1/23 daemons upgraded", 2024-02-07T12:25:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "", 2024-02-07T12:25:00.280 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:25:00.280 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:25:00.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:00 smithi184 ceph-mon[27098]: from='client.24475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:00.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:00 smithi184 ceph-mon[27098]: from='client.24479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:00.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:00 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/4119637621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:25:00.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 107706 13.68 MB/sec execute 48 sec latency 57.912 ms 2024-02-07T12:25:01.021 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:25:01.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:01 smithi174 ceph-mon[24580]: pgmap v26: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 584 op/s 2024-02-07T12:25:01.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:01 smithi174 ceph-mon[24580]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:01.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:01 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/1648136461' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:25:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:01 smithi184 ceph-mon[27098]: pgmap v26: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 584 op/s 2024-02-07T12:25:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:01 smithi184 ceph-mon[27098]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:01 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/1648136461' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:25:01.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 108100 13.59 MB/sec execute 49 sec latency 38.997 ms 2024-02-07T12:25:02.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 108495 13.45 MB/sec execute 50 sec latency 41.484 ms 2024-02-07T12:25:03.242 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:03 smithi174 ceph-mon[24580]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 735 op/s 2024-02-07T12:25:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:03 smithi184 ceph-mon[27098]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 735 op/s 2024-02-07T12:25:03.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 108831 13.34 MB/sec execute 51 sec latency 57.677 ms 2024-02-07T12:25:04.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 109256 13.37 MB/sec execute 52 sec latency 91.870 ms 2024-02-07T12:25:05.263 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:05 smithi174 ceph-mon[24580]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 456 op/s 2024-02-07T12:25:05.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:05 smithi184 ceph-mon[27098]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 456 op/s 2024-02-07T12:25:05.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 109664 13.40 MB/sec execute 53 sec latency 49.138 ms 2024-02-07T12:25:06.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 110091 13.50 MB/sec execute 54 sec latency 40.908 ms 2024-02-07T12:25:07.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:07 smithi174 ceph-mon[24580]: pgmap v29: 97 pgs: 97 active+clean; 325 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 678 op/s 2024-02-07T12:25:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:07 smithi184 ceph-mon[27098]: pgmap v29: 97 pgs: 97 active+clean; 325 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 678 op/s 2024-02-07T12:25:07.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 110391 13.39 MB/sec execute 55 sec latency 51.152 ms 2024-02-07T12:25:08.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 110898 13.30 MB/sec execute 56 sec latency 23.718 ms 2024-02-07T12:25:09.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:09 smithi174 ceph-mon[24580]: pgmap v30: 97 pgs: 97 active+clean; 325 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 443 op/s 2024-02-07T12:25:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:09 smithi184 ceph-mon[27098]: pgmap v30: 97 pgs: 97 active+clean; 325 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 443 op/s 2024-02-07T12:25:09.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 111493 13.27 MB/sec execute 57 sec latency 30.363 ms 2024-02-07T12:25:10.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 112243 13.33 MB/sec execute 58 sec latency 24.728 ms 2024-02-07T12:25:11.287 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:11 smithi174 ceph-mon[24580]: pgmap v31: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.4 MiB/s wr, 553 op/s 2024-02-07T12:25:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:11 smithi184 ceph-mon[27098]: pgmap v31: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.4 MiB/s wr, 553 op/s 2024-02-07T12:25:11.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 112803 13.40 MB/sec execute 59 sec latency 29.944 ms 2024-02-07T12:25:12.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 113420 13.61 MB/sec execute 60 sec latency 22.265 ms 2024-02-07T12:25:13.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:13 smithi174 ceph-mon[24580]: pgmap v32: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 659 op/s 2024-02-07T12:25:13.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:13 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:13.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:13 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:13 smithi184 ceph-mon[27098]: pgmap v32: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 659 op/s 2024-02-07T12:25:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:13 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:13 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:13.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 113850 13.62 MB/sec execute 61 sec latency 34.354 ms 2024-02-07T12:25:14.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:14 smithi174 ceph-mon[24580]: Reconfiguring node-exporter.smithi174 (dependencies changed)... 2024-02-07T12:25:14.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:14 smithi174 ceph-mon[24580]: Reconfiguring daemon node-exporter.smithi174 on smithi174 2024-02-07T12:25:14.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:25:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:14 smithi184 ceph-mon[27098]: Reconfiguring node-exporter.smithi174 (dependencies changed)... 2024-02-07T12:25:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:14 smithi184 ceph-mon[27098]: Reconfiguring daemon node-exporter.smithi174 on smithi174 2024-02-07T12:25:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:25:14.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 114134 13.46 MB/sec execute 62 sec latency 34.238 ms 2024-02-07T12:25:15.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:15 smithi174 ceph-mon[24580]: pgmap v33: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 489 op/s 2024-02-07T12:25:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:15 smithi184 ceph-mon[27098]: pgmap v33: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 489 op/s 2024-02-07T12:25:15.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 114439 13.33 MB/sec execute 63 sec latency 39.200 ms 2024-02-07T12:25:16.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 114782 13.19 MB/sec execute 64 sec latency 62.076 ms 2024-02-07T12:25:17.617 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:17 smithi174 ceph-mon[24580]: pgmap v34: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 806 op/s 2024-02-07T12:25:17.617 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:17 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:17.618 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:17 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:17.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:17 smithi184 ceph-mon[27098]: pgmap v34: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 806 op/s 2024-02-07T12:25:17.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:17 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:17.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:17 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:17.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 115206 13.20 MB/sec execute 65 sec latency 41.558 ms 2024-02-07T12:25:18.632 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:18 smithi174 ceph-mon[24580]: Reconfiguring prometheus.smithi174 (dependencies changed)... 2024-02-07T12:25:18.632 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:18 smithi174 ceph-mon[24580]: Reconfiguring daemon prometheus.smithi174 on smithi174 2024-02-07T12:25:18.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:18 smithi184 ceph-mon[27098]: Reconfiguring prometheus.smithi174 (dependencies changed)... 2024-02-07T12:25:18.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:18 smithi184 ceph-mon[27098]: Reconfiguring daemon prometheus.smithi174 on smithi174 2024-02-07T12:25:18.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 115699 13.15 MB/sec execute 66 sec latency 37.788 ms 2024-02-07T12:25:19.645 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:19 smithi174 ceph-mon[24580]: pgmap v35: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 585 op/s 2024-02-07T12:25:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:19 smithi184 ceph-mon[27098]: pgmap v35: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 585 op/s 2024-02-07T12:25:19.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 116159 13.12 MB/sec execute 67 sec latency 71.935 ms 2024-02-07T12:25:20.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 116649 13.21 MB/sec execute 68 sec latency 37.092 ms 2024-02-07T12:25:21.668 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:21 smithi174 ceph-mon[24580]: pgmap v36: 97 pgs: 97 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.4 MiB/s wr, 597 op/s 2024-02-07T12:25:21.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:21 smithi184 ceph-mon[27098]: pgmap v36: 97 pgs: 97 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.4 MiB/s wr, 597 op/s 2024-02-07T12:25:21.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 117065 13.27 MB/sec execute 69 sec latency 46.327 ms 2024-02-07T12:25:22.677 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:22 smithi174 ceph-mon[24580]: pgmap v37: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 725 op/s 2024-02-07T12:25:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:22 smithi184 ceph-mon[27098]: pgmap v37: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 725 op/s 2024-02-07T12:25:22.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 117455 13.25 MB/sec execute 70 sec latency 36.377 ms 2024-02-07T12:25:23.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 117690 13.11 MB/sec execute 71 sec latency 127.340 ms 2024-02-07T12:25:24.950 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:24 smithi174 ceph-mon[24580]: pgmap v38: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 566 op/s 2024-02-07T12:25:24.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 117955 12.99 MB/sec execute 72 sec latency 60.322 ms 2024-02-07T12:25:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:24 smithi184 ceph-mon[27098]: pgmap v38: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 566 op/s 2024-02-07T12:25:25.995 INFO:teuthology.orchestra.run.smithi174.stdout: 5 118511 12.98 MB/sec execute 73 sec latency 37.432 ms 2024-02-07T12:25:26.996 INFO:teuthology.orchestra.run.smithi174.stdout: 5 118930 12.90 MB/sec execute 74 sec latency 37.841 ms 2024-02-07T12:25:27.222 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:27 smithi174 ceph-mon[24580]: pgmap v39: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 786 op/s 2024-02-07T12:25:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:27 smithi184 ceph-mon[27098]: pgmap v39: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 786 op/s 2024-02-07T12:25:27.996 INFO:teuthology.orchestra.run.smithi174.stdout: 5 119441 12.89 MB/sec execute 75 sec latency 37.018 ms 2024-02-07T12:25:29.000 INFO:teuthology.orchestra.run.smithi174.stdout: 5 120007 12.98 MB/sec execute 76 sec latency 40.250 ms 2024-02-07T12:25:29.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:29 smithi174 ceph-mon[24580]: pgmap v40: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 469 op/s 2024-02-07T12:25:29.244 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:29 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:25:29.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:29 smithi184 ceph-mon[27098]: pgmap v40: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 469 op/s 2024-02-07T12:25:29.393 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:29 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:25:30.007 INFO:teuthology.orchestra.run.smithi174.stdout: 5 120845 13.22 MB/sec execute 77 sec latency 24.862 ms 2024-02-07T12:25:31.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 121179 13.15 MB/sec execute 78 sec latency 64.143 ms 2024-02-07T12:25:31.265 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:31 smithi174 ceph-mon[24580]: pgmap v41: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 583 op/s 2024-02-07T12:25:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:31 smithi184 ceph-mon[27098]: pgmap v41: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 583 op/s 2024-02-07T12:25:31.676 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:25:32.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 121504 13.05 MB/sec execute 79 sec latency 49.532 ms 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 starting - - - - 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (9m) 63s ago 9m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (9m) 66s ago 9m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 starting - - - - 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (3m) 63s ago 3m 36.9M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (3m) 66s ago 3m 11.7M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:9283 running (11m) 63s ago 11m 325M - 17.2.0 e1d6a67b021e be683bef83c5 2024-02-07T12:25:32.689 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (88s) 66s ago 9m 487M - 19.0.0-1252-g44458cce 79957b372307 775a57400e80 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (11m) 63s ago 11m 48.2M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (9m) 66s ago 9m 38.5M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (3m) 63s ago 3m 253M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (3m) 66s ago 3m 74.0M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 starting - - - - 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (8m) 66s ago 8m 17.3M - 1.3.1 1dbe0e931976 7e6cdb451097 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (8m) 63s ago 8m 341M 1936M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (7m) 63s ago 7m 262M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (7m) 63s ago 7m 206M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (6m) 63s ago 6m 295M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (6m) 66s ago 6m 329M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (6m) 66s ago 6m 346M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (5m) 66s ago 5m 273M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:25:32.690 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (5m) 66s ago 5m 435M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:25:32.691 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (8m) 63s ago 8m 65.6M - 2.33.4 514e6a882f6e 90c30949c773 2024-02-07T12:25:33.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 121903 12.94 MB/sec execute 80 sec latency 44.077 ms 2024-02-07T12:25:33.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:33 smithi174 ceph-mon[24580]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:33.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:33 smithi174 ceph-mon[24580]: pgmap v42: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 792 op/s 2024-02-07T12:25:33.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:33 smithi174 ceph-mon[24580]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:33.305 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:25:33.305 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:25:33.306 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-02-07T12:25:33.307 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:25:33.307 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:25:33.307 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:25:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:33 smithi184 ceph-mon[27098]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:33 smithi184 ceph-mon[27098]: pgmap v42: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 792 op/s 2024-02-07T12:25:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:33 smithi184 ceph-mon[27098]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:34.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 122282 12.95 MB/sec execute 81 sec latency 44.931 ms 2024-02-07T12:25:34.031 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:25:34.032 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:25:34.032 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:25:34.032 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:25:34.032 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [], 2024-02-07T12:25:34.032 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "1/23 daemons upgraded", 2024-02-07T12:25:34.032 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "", 2024-02-07T12:25:34.032 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:25:34.032 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:25:34.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:34 smithi174 ceph-mon[24580]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:34.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:34 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3635517751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:25:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:34 smithi184 ceph-mon[27098]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:34 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3635517751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:25:34.650 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:25:35.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 123069 12.98 MB/sec execute 82 sec latency 30.984 ms 2024-02-07T12:25:35.248 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:35 smithi174 ceph-mon[24580]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:35.249 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:35 smithi174 ceph-mon[24580]: pgmap v43: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.1 MiB/s wr, 555 op/s 2024-02-07T12:25:35.249 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:35 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3144172306' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:25:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:35 smithi184 ceph-mon[27098]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:25:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:35 smithi184 ceph-mon[27098]: pgmap v43: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.1 MiB/s wr, 555 op/s 2024-02-07T12:25:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:35 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3144172306' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:25:36.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 124009 13.25 MB/sec execute 83 sec latency 16.425 ms 2024-02-07T12:25:37.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 124682 13.33 MB/sec execute 84 sec latency 31.752 ms 2024-02-07T12:25:37.261 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:37 smithi174 ceph-mon[24580]: pgmap v44: 97 pgs: 97 active+clean; 323 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 829 op/s 2024-02-07T12:25:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:37 smithi184 ceph-mon[27098]: pgmap v44: 97 pgs: 97 active+clean; 323 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 829 op/s 2024-02-07T12:25:38.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 125128 13.26 MB/sec execute 85 sec latency 43.655 ms 2024-02-07T12:25:39.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 125539 13.21 MB/sec execute 86 sec latency 47.983 ms 2024-02-07T12:25:39.293 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:39 smithi174 ceph-mon[24580]: pgmap v45: 97 pgs: 97 active+clean; 323 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 609 op/s 2024-02-07T12:25:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:39 smithi184 ceph-mon[27098]: pgmap v45: 97 pgs: 97 active+clean; 323 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 609 op/s 2024-02-07T12:25:40.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 125917 13.16 MB/sec execute 87 sec latency 44.250 ms 2024-02-07T12:25:41.012 INFO:teuthology.orchestra.run.smithi174.stdout: 5 126528 13.16 MB/sec execute 88 sec latency 56.245 ms 2024-02-07T12:25:41.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:41 smithi174 ceph-mon[24580]: pgmap v46: 97 pgs: 97 active+clean; 323 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 6.8 MiB/s wr, 729 op/s 2024-02-07T12:25:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:41 smithi184 ceph-mon[27098]: pgmap v46: 97 pgs: 97 active+clean; 323 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 6.8 MiB/s wr, 729 op/s 2024-02-07T12:25:42.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 127045 13.21 MB/sec execute 89 sec latency 25.050 ms 2024-02-07T12:25:43.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 127441 13.20 MB/sec execute 90 sec latency 58.479 ms 2024-02-07T12:25:43.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:43 smithi174 ceph-mon[24580]: pgmap v47: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.0 MiB/s wr, 849 op/s 2024-02-07T12:25:43.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:43 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:43.348 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:43 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:43 smithi184 ceph-mon[27098]: pgmap v47: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.0 MiB/s wr, 849 op/s 2024-02-07T12:25:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:43 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:43 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:44.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 127852 13.26 MB/sec execute 91 sec latency 73.975 ms 2024-02-07T12:25:44.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:44 smithi174 ceph-mon[24580]: Reconfiguring node-exporter.smithi184 (dependencies changed)... 2024-02-07T12:25:44.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:44 smithi174 ceph-mon[24580]: Reconfiguring daemon node-exporter.smithi184 on smithi184 2024-02-07T12:25:44.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:44 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:25:44.434 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:44 smithi184 ceph-mon[27098]: Reconfiguring node-exporter.smithi184 (dependencies changed)... 2024-02-07T12:25:44.434 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:44 smithi184 ceph-mon[27098]: Reconfiguring daemon node-exporter.smithi184 on smithi184 2024-02-07T12:25:44.434 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:44 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:25:45.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 128173 13.21 MB/sec execute 92 sec latency 50.001 ms 2024-02-07T12:25:45.239 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:45 smithi184 ceph-mon[27098]: pgmap v48: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 628 op/s 2024-02-07T12:25:45.385 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:45 smithi174 ceph-mon[24580]: pgmap v48: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 628 op/s 2024-02-07T12:25:46.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 128527 13.13 MB/sec execute 93 sec latency 59.142 ms 2024-02-07T12:25:46.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: mgrmap e27: smithi184.prgadw(active, since 91s), standbys: smithi174.whsozo 2024-02-07T12:25:46.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:25:46.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:25:46.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi174.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:25:46.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi174.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:25:46.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi174.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi174.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi174.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi174.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:46 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:25:46.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: mgrmap e27: smithi184.prgadw(active, since 91s), standbys: smithi174.whsozo 2024-02-07T12:25:46.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi174.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi174.front.sepia.ceph.com:9093"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi174.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi174.front.sepia.ceph.com:3000"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi174.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi174.front.sepia.ceph.com:9095"}]: dispatch 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:46.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:46 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:25:47.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 128960 13.05 MB/sec execute 94 sec latency 39.499 ms 2024-02-07T12:25:47.661 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:47 smithi174 ceph-mon[24580]: Upgrade: Need to upgrade myself (mgr.smithi184.prgadw) 2024-02-07T12:25:47.662 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:47 smithi174 ceph-mon[24580]: Upgrade: Need to upgrade myself (mgr.smithi184.prgadw) 2024-02-07T12:25:47.662 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:47 smithi174 ceph-mon[24580]: pgmap v49: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 885 op/s 2024-02-07T12:25:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:47 smithi184 ceph-mon[27098]: Upgrade: Need to upgrade myself (mgr.smithi184.prgadw) 2024-02-07T12:25:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:47 smithi184 ceph-mon[27098]: Upgrade: Need to upgrade myself (mgr.smithi184.prgadw) 2024-02-07T12:25:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:47 smithi184 ceph-mon[27098]: pgmap v49: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 885 op/s 2024-02-07T12:25:48.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 129433 13.06 MB/sec execute 95 sec latency 62.627 ms 2024-02-07T12:25:49.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 129899 13.03 MB/sec execute 96 sec latency 45.737 ms 2024-02-07T12:25:49.423 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:49 smithi174 ceph-mon[24580]: pgmap v50: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.8 MiB/s wr, 611 op/s 2024-02-07T12:25:49.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:49 smithi174 ceph-mon[24580]: Upgrade: Updating mgr.smithi174.whsozo 2024-02-07T12:25:49.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:49 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:49.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:49 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.whsozo", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:25:49.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:49 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.whsozo", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:25:49.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:49 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:25:49.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:49 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:25:49.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:49 smithi174 ceph-mon[24580]: Deploying daemon mgr.smithi174.whsozo on smithi174 2024-02-07T12:25:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:49 smithi184 ceph-mon[27098]: pgmap v50: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.8 MiB/s wr, 611 op/s 2024-02-07T12:25:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:49 smithi184 ceph-mon[27098]: Upgrade: Updating mgr.smithi174.whsozo 2024-02-07T12:25:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:49 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:49 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.whsozo", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:25:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:49 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.whsozo", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:25:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:49 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:25:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:49 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:25:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:49 smithi184 ceph-mon[27098]: Deploying daemon mgr.smithi174.whsozo on smithi174 2024-02-07T12:25:50.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 130256 12.99 MB/sec execute 97 sec latency 50.583 ms 2024-02-07T12:25:51.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 130606 12.98 MB/sec execute 98 sec latency 38.379 ms 2024-02-07T12:25:51.470 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:51 smithi174 ceph-mon[24580]: pgmap v51: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 730 op/s 2024-02-07T12:25:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:51 smithi184 ceph-mon[27098]: pgmap v51: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 730 op/s 2024-02-07T12:25:52.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 130935 12.96 MB/sec execute 99 sec latency 39.550 ms 2024-02-07T12:25:53.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 131309 12.97 MB/sec execute 100 sec latency 51.322 ms 2024-02-07T12:25:53.246 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:53 smithi174 ceph-mon[24580]: pgmap v52: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.9 MiB/s wr, 751 op/s 2024-02-07T12:25:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:53 smithi184 ceph-mon[27098]: pgmap v52: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.9 MiB/s wr, 751 op/s 2024-02-07T12:25:54.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 131588 12.93 MB/sec execute 101 sec latency 45.072 ms 2024-02-07T12:25:55.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 131823 12.87 MB/sec execute 102 sec latency 53.919 ms 2024-02-07T12:25:55.289 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:55 smithi174 ceph-mon[24580]: pgmap v53: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 517 op/s 2024-02-07T12:25:55.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:55 smithi184 ceph-mon[27098]: pgmap v53: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 517 op/s 2024-02-07T12:25:56.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 132142 12.79 MB/sec execute 103 sec latency 47.486 ms 2024-02-07T12:25:57.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 132434 12.69 MB/sec execute 104 sec latency 48.111 ms 2024-02-07T12:25:57.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:57 smithi174 ceph-mon[24580]: pgmap v54: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.6 MiB/s wr, 710 op/s 2024-02-07T12:25:57.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:57 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:57.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:57 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:57.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:57 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:25:57.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:57 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:57.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:57 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:57 smithi184 ceph-mon[27098]: pgmap v54: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.6 MiB/s wr, 710 op/s 2024-02-07T12:25:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:57 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:57.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:57 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:57.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:57 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:25:57.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:57 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:57.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:57 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:25:58.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 132684 12.65 MB/sec execute 105 sec latency 37.917 ms 2024-02-07T12:25:59.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 133180 12.66 MB/sec execute 106 sec latency 38.206 ms 2024-02-07T12:25:59.329 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:59 smithi174 ceph-mon[24580]: pgmap v55: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 452 op/s 2024-02-07T12:25:59.330 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:25:59 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:25:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:59 smithi184 ceph-mon[27098]: pgmap v55: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 452 op/s 2024-02-07T12:25:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:25:59 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:00.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 133647 12.62 MB/sec execute 107 sec latency 36.102 ms 2024-02-07T12:26:01.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 134052 12.61 MB/sec execute 108 sec latency 73.318 ms 2024-02-07T12:26:01.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:01 smithi174 ceph-mon[24580]: pgmap v56: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 477 op/s 2024-02-07T12:26:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:01 smithi184 ceph-mon[27098]: pgmap v56: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 477 op/s 2024-02-07T12:26:02.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 134336 12.61 MB/sec execute 109 sec latency 41.965 ms 2024-02-07T12:26:02.345 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:02 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:02.345 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:02 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:02 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:02 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:03.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 134722 12.61 MB/sec execute 110 sec latency 43.523 ms 2024-02-07T12:26:03.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:03 smithi174 ceph-mon[24580]: pgmap v57: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 557 op/s 2024-02-07T12:26:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:03 smithi184 ceph-mon[27098]: pgmap v57: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 557 op/s 2024-02-07T12:26:04.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 134968 12.57 MB/sec execute 111 sec latency 42.747 ms 2024-02-07T12:26:05.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 135193 12.55 MB/sec execute 112 sec latency 60.557 ms 2024-02-07T12:26:05.128 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:26:05.418 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:05 smithi174 ceph-mon[24580]: pgmap v58: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-02-07T12:26:05.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:05 smithi184 ceph-mon[27098]: pgmap v58: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-02-07T12:26:05.983 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:26:05.983 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 starting - - - - 2024-02-07T12:26:05.983 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (10m) 96s ago 10m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:26:05.983 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (9m) 4s ago 9m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:26:05.983 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 starting - - - - 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (4m) 96s ago 4m 36.9M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (4m) 4s ago 4m 12.2M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 starting - - - - 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (2m) 4s ago 9m 502M - 19.0.0-1252-g44458cce 79957b372307 775a57400e80 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (11m) 96s ago 11m 48.2M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (9m) 4s ago 9m 40.7M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (4m) 96s ago 4m 253M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (4m) 4s ago 4m 79.9M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 starting - - - - 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (21s) 4s ago 9m 10.6M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (8m) 96s ago 8m 341M 1936M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (8m) 96s ago 8m 262M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:26:05.984 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (7m) 96s ago 7m 206M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:26:05.985 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (7m) 96s ago 7m 295M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:26:05.985 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (7m) 4s ago 7m 449M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:26:05.985 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (6m) 4s ago 6m 497M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:26:05.985 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (6m) 4s ago 6m 415M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:26:05.985 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (5m) 4s ago 5m 584M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:26:05.985 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 starting - - - - 2024-02-07T12:26:06.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 135432 12.49 MB/sec execute 113 sec latency 45.563 ms 2024-02-07T12:26:06.448 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: from='client.14786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:06.448 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: Standby manager daemon smithi174.whsozo restarted 2024-02-07T12:26:06.448 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: Standby manager daemon smithi174.whsozo started 2024-02-07T12:26:06.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.whsozo/crt"}]: dispatch 2024-02-07T12:26:06.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:26:06.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.whsozo/key"}]: dispatch 2024-02-07T12:26:06.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:26:06.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:06.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:06 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:06.563 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:26:06.563 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:26:06.563 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:26:06.563 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:26:06.563 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:26:06.563 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:26:06.563 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:26:06.563 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:26:06.564 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:26:06.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: from='client.14786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:06.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: Standby manager daemon smithi174.whsozo restarted 2024-02-07T12:26:06.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: Standby manager daemon smithi174.whsozo started 2024-02-07T12:26:06.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.whsozo/crt"}]: dispatch 2024-02-07T12:26:06.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:26:06.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.whsozo/key"}]: dispatch 2024-02-07T12:26:06.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:26:06.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:06.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:06 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:07.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 135687 12.39 MB/sec execute 114 sec latency 60.747 ms 2024-02-07T12:26:07.122 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "2/23 daemons upgraded", 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading mgr daemons", 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:26:07.123 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:26:07.474 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:07 smithi174 ceph-mon[24580]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:07.474 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:07 smithi174 ceph-mon[24580]: pgmap v59: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.5 MiB/s wr, 603 op/s 2024-02-07T12:26:07.474 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:07 smithi174 ceph-mon[24580]: from='client.14794 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:07.474 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:07 smithi174 ceph-mon[24580]: mgrmap e28: smithi184.prgadw(active, since 112s), standbys: smithi174.whsozo 2024-02-07T12:26:07.474 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:07 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/876786865' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:26:07.694 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:26:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:07 smithi184 ceph-mon[27098]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:07 smithi184 ceph-mon[27098]: pgmap v59: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.5 MiB/s wr, 603 op/s 2024-02-07T12:26:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:07 smithi184 ceph-mon[27098]: from='client.14794 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:07 smithi184 ceph-mon[27098]: mgrmap e28: smithi184.prgadw(active, since 112s), standbys: smithi174.whsozo 2024-02-07T12:26:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:07 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/876786865' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:26:08.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 135993 12.32 MB/sec execute 115 sec latency 62.659 ms 2024-02-07T12:26:08.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:08 smithi174 ceph-mon[24580]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:08.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:08 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:08.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:08 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:08.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:08 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/3503315339' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:26:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:08 smithi184 ceph-mon[27098]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:08 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:08 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:08 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3503315339' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:26:09.013 INFO:teuthology.orchestra.run.smithi174.stdout: 5 136485 12.35 MB/sec execute 116 sec latency 47.900 ms 2024-02-07T12:26:09.333 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:09 smithi174 ceph-mon[24580]: pgmap v60: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-02-07T12:26:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:09 smithi184 ceph-mon[27098]: pgmap v60: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-02-07T12:26:10.014 INFO:teuthology.orchestra.run.smithi174.stdout: 5 137141 12.37 MB/sec execute 117 sec latency 37.548 ms 2024-02-07T12:26:11.014 INFO:teuthology.orchestra.run.smithi174.stdout: 5 137587 12.35 MB/sec execute 118 sec latency 46.765 ms 2024-02-07T12:26:11.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:11 smithi174 ceph-mon[24580]: pgmap v61: 97 pgs: 97 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.8 MiB/s wr, 504 op/s 2024-02-07T12:26:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:11 smithi184 ceph-mon[27098]: pgmap v61: 97 pgs: 97 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.8 MiB/s wr, 504 op/s 2024-02-07T12:26:12.018 INFO:teuthology.orchestra.run.smithi174.stdout: 5 137835 12.32 MB/sec execute 119 sec latency 39.081 ms 2024-02-07T12:26:13.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 138184 12.34 MB/sec execute 120 sec latency 58.304 ms 2024-02-07T12:26:13.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:13 smithi174 ceph-mon[24580]: pgmap v62: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.9 MiB/s wr, 622 op/s 2024-02-07T12:26:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:13 smithi184 ceph-mon[27098]: pgmap v62: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.9 MiB/s wr, 622 op/s 2024-02-07T12:26:14.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 138433 12.31 MB/sec execute 121 sec latency 47.281 ms 2024-02-07T12:26:14.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:14.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:14.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:14.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:26:14.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:14.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:14.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:14.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr fail", "who": "smithi184.prgadw"}]: dispatch 2024-02-07T12:26:14.647 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr fail", "who": "smithi184.prgadw"}]: dispatch 2024-02-07T12:26:14.647 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: osdmap e72: 8 total, 8 up, 8 in 2024-02-07T12:26:14.647 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:14 smithi174 ceph-mon[24580]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr fail", "who": "smithi184.prgadw"}]: dispatch 2024-02-07T12:26:14.756 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "mgr fail", "who": "smithi184.prgadw"}]: dispatch 2024-02-07T12:26:14.757 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: osdmap e72: 8 total, 8 up, 8 in 2024-02-07T12:26:14.757 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:14 smithi184 ceph-mon[27098]: from='mgr.24419 172.21.15.184:0/2212995274' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:15.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 138652 12.27 MB/sec execute 122 sec latency 57.398 ms 2024-02-07T12:26:15.672 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: Upgrade: Need to upgrade myself (mgr.smithi184.prgadw) 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: Upgrade: Need to upgrade myself (mgr.smithi184.prgadw) 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: Failing over to other MGR 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: pgmap v64: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd='[{"prefix": "mgr fail", "who": "smithi184.prgadw"}]': finished 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: mgrmap e29: smithi174.whsozo(active, starting, since 0.866243s), standbys: smithi184.prgadw 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi184.prgadw", "id": "smithi184.prgadw"}]: dispatch 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:26:15.673 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:26:15.674 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:15 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:26:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: Upgrade: Need to upgrade myself (mgr.smithi184.prgadw) 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: Upgrade: Need to upgrade myself (mgr.smithi184.prgadw) 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: Failing over to other MGR 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: pgmap v64: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.24419 ' entity='mgr.smithi184.prgadw' cmd='[{"prefix": "mgr fail", "who": "smithi184.prgadw"}]': finished 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: mgrmap e29: smithi174.whsozo(active, starting, since 0.866243s), standbys: smithi184.prgadw 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi184.prgadw", "id": "smithi184.prgadw"}]: dispatch 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:26:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:15.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:26:15.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:15 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:26:16.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 138905 12.24 MB/sec execute 123 sec latency 45.874 ms 2024-02-07T12:26:16.703 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:16 smithi174 ceph-mon[24580]: [07/Feb/2024:12:26:15] ENGINE Bus STARTING 2024-02-07T12:26:16.704 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:16 smithi174 ceph-mon[24580]: [07/Feb/2024:12:26:15] ENGINE Serving on http://172.21.15.174:8765 2024-02-07T12:26:16.704 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:16 smithi174 ceph-mon[24580]: [07/Feb/2024:12:26:15] ENGINE Serving on https://172.21.15.174:7150 2024-02-07T12:26:16.704 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:16 smithi174 ceph-mon[24580]: [07/Feb/2024:12:26:15] ENGINE Bus STARTED 2024-02-07T12:26:16.704 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:16 smithi174 ceph-mon[24580]: mgrmap e30: smithi174.whsozo(active, since 1.87225s), standbys: smithi184.prgadw 2024-02-07T12:26:16.704 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:16 smithi174 ceph-mon[24580]: pgmap v3: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:26:16.704 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:16 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:16.704 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:16 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:16.760 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:16 smithi184 ceph-mon[27098]: [07/Feb/2024:12:26:15] ENGINE Bus STARTING 2024-02-07T12:26:16.760 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:16 smithi184 ceph-mon[27098]: [07/Feb/2024:12:26:15] ENGINE Serving on http://172.21.15.174:8765 2024-02-07T12:26:16.760 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:16 smithi184 ceph-mon[27098]: [07/Feb/2024:12:26:15] ENGINE Serving on https://172.21.15.174:7150 2024-02-07T12:26:16.760 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:16 smithi184 ceph-mon[27098]: [07/Feb/2024:12:26:15] ENGINE Bus STARTED 2024-02-07T12:26:16.760 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:16 smithi184 ceph-mon[27098]: mgrmap e30: smithi174.whsozo(active, since 1.87225s), standbys: smithi184.prgadw 2024-02-07T12:26:16.760 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:16 smithi184 ceph-mon[27098]: pgmap v3: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:26:16.760 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:16 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:16.760 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:16 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:17.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 139171 12.16 MB/sec execute 124 sec latency 75.768 ms 2024-02-07T12:26:17.596 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:17 smithi184 ceph-mon[27098]: pgmap v4: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:26:17.712 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:17 smithi174 ceph-mon[24580]: pgmap v4: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:26:18.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 139436 12.09 MB/sec execute 125 sec latency 68.816 ms 2024-02-07T12:26:18.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:18 smithi184 ceph-mon[27098]: mgrmap e31: smithi174.whsozo(active, since 3s), standbys: smithi184.prgadw 2024-02-07T12:26:18.746 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:18 smithi174 ceph-mon[24580]: mgrmap e31: smithi174.whsozo(active, since 3s), standbys: smithi184.prgadw 2024-02-07T12:26:19.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 139665 12.02 MB/sec execute 126 sec latency 49.178 ms 2024-02-07T12:26:19.688 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:19 smithi184 ceph-mon[27098]: pgmap v5: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:26:19.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:19 smithi174 ceph-mon[24580]: pgmap v5: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:26:20.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 139981 12.02 MB/sec execute 127 sec latency 56.511 ms 2024-02-07T12:26:21.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 140302 11.98 MB/sec execute 128 sec latency 68.553 ms 2024-02-07T12:26:21.039 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:20 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:21.039 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:20 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:21.039 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:20 smithi174 ceph-mon[24580]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 519 KiB/s wr, 39 op/s 2024-02-07T12:26:21.195 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:20 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:21.195 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:20 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:21.196 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:20 smithi184 ceph-mon[27098]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 519 KiB/s wr, 39 op/s 2024-02-07T12:26:22.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 140674 11.96 MB/sec execute 129 sec latency 51.739 ms 2024-02-07T12:26:22.563 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:22 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:22.564 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:22 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:22 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:22 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:23.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 141038 11.92 MB/sec execute 130 sec latency 37.977 ms 2024-02-07T12:26:23.581 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:23 smithi174 ceph-mon[24580]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 2.7 MiB/s wr, 328 op/s 2024-02-07T12:26:23.582 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:23 smithi174 ceph-mon[24580]: Standby manager daemon smithi184.prgadw restarted 2024-02-07T12:26:23.582 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:23 smithi174 ceph-mon[24580]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:26:23.582 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:23 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/2318735135' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:26:23.582 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:23 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/2318735135' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:26:23.582 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:23 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/2318735135' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:26:23.582 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:23 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/2318735135' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:26:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:23 smithi184 ceph-mon[27098]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 2.7 MiB/s wr, 328 op/s 2024-02-07T12:26:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:23 smithi184 ceph-mon[27098]: Standby manager daemon smithi184.prgadw restarted 2024-02-07T12:26:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:23 smithi184 ceph-mon[27098]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:26:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:23 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/2318735135' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:26:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:23 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/2318735135' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:26:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:23 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/2318735135' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:26:23.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:23 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/2318735135' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:26:24.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 141352 11.92 MB/sec execute 131 sec latency 46.797 ms 2024-02-07T12:26:24.633 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:24 smithi174 ceph-mon[24580]: mgrmap e32: smithi174.whsozo(active, since 9s), standbys: smithi184.prgadw 2024-02-07T12:26:24.633 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:24 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:24.634 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:24 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:24 smithi184 ceph-mon[27098]: mgrmap e32: smithi174.whsozo(active, since 9s), standbys: smithi184.prgadw 2024-02-07T12:26:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:24 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:24 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:25.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 141749 11.95 MB/sec execute 132 sec latency 40.323 ms 2024-02-07T12:26:25.655 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:25 smithi174 ceph-mon[24580]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 2.1 MiB/s wr, 254 op/s 2024-02-07T12:26:25.655 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:25 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:25.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:25 smithi184 ceph-mon[27098]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 2.1 MiB/s wr, 254 op/s 2024-02-07T12:26:25.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:25 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:26.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 142129 11.95 MB/sec execute 133 sec latency 46.804 ms 2024-02-07T12:26:26.415 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:26 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:26.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:26 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:27.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 142660 11.96 MB/sec execute 134 sec latency 33.306 ms 2024-02-07T12:26:27.419 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:27 smithi174 ceph-mon[24580]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 428 op/s 2024-02-07T12:26:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:27 smithi184 ceph-mon[27098]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 428 op/s 2024-02-07T12:26:28.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 143151 11.92 MB/sec execute 135 sec latency 31.577 ms 2024-02-07T12:26:29.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 143678 11.95 MB/sec execute 136 sec latency 47.668 ms 2024-02-07T12:26:29.693 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:29 smithi174 ceph-mon[24580]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.7 MiB/s wr, 391 op/s 2024-02-07T12:26:29.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:29 smithi184 ceph-mon[27098]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.7 MiB/s wr, 391 op/s 2024-02-07T12:26:30.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 144323 11.97 MB/sec execute 137 sec latency 34.847 ms 2024-02-07T12:26:30.698 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:30 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:30.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:30 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:31.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 144864 12.00 MB/sec execute 138 sec latency 35.490 ms 2024-02-07T12:26:31.704 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:31 smithi174 ceph-mon[24580]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.7 MiB/s wr, 538 op/s 2024-02-07T12:26:31.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:31 smithi184 ceph-mon[27098]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.7 MiB/s wr, 538 op/s 2024-02-07T12:26:32.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 145418 12.05 MB/sec execute 139 sec latency 40.852 ms 2024-02-07T12:26:33.020 INFO:teuthology.orchestra.run.smithi174.stdout: 5 145961 12.11 MB/sec execute 140 sec latency 28.365 ms 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: Updating smithi184:/etc/ceph/ceph.conf 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 6.5 MiB/s wr, 705 op/s 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:26:33.426 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:33 smithi174 ceph-mon[24580]: Updating smithi184:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:26:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:26:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:26:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: Updating smithi184:/etc/ceph/ceph.conf 2024-02-07T12:26:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 6.5 MiB/s wr, 705 op/s 2024-02-07T12:26:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:26:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:26:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:26:33.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:33 smithi184 ceph-mon[27098]: Updating smithi184:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:26:34.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 146488 12.07 MB/sec execute 141 sec latency 26.293 ms 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.682 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:34 smithi174 ceph-mon[24580]: Reconfiguring prometheus.smithi174 (dependencies changed)... 2024-02-07T12:26:34.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:26:34.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:26:34.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:34.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:34 smithi184 ceph-mon[27098]: Reconfiguring prometheus.smithi174 (dependencies changed)... 2024-02-07T12:26:35.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 147082 12.11 MB/sec execute 142 sec latency 25.464 ms 2024-02-07T12:26:35.685 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:35 smithi174 ceph-mon[24580]: Reconfiguring daemon prometheus.smithi174 on smithi174 2024-02-07T12:26:35.686 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:35 smithi174 ceph-mon[24580]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 531 op/s 2024-02-07T12:26:35.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:35 smithi184 ceph-mon[27098]: Reconfiguring daemon prometheus.smithi174 on smithi174 2024-02-07T12:26:35.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:35 smithi184 ceph-mon[27098]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 531 op/s 2024-02-07T12:26:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 147817 12.13 MB/sec execute 143 sec latency 31.983 ms 2024-02-07T12:26:37.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 148319 12.13 MB/sec execute 144 sec latency 40.100 ms 2024-02-07T12:26:37.205 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:37 smithi174 ceph-mon[24580]: pgmap v14: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 876 op/s 2024-02-07T12:26:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:37 smithi184 ceph-mon[27098]: pgmap v14: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 876 op/s 2024-02-07T12:26:38.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 148854 12.20 MB/sec execute 145 sec latency 34.312 ms 2024-02-07T12:26:38.256 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:26:39.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 149418 12.26 MB/sec execute 146 sec latency 38.206 ms 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (109s) 15s ago 10m 23.8M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (10m) 15s ago 10m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (10m) 19s ago 10m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (87s) 15s ago 10m 44.7M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (5m) 15s ago 5m 37.6M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (4m) 19s ago 4m 12.4M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (43s) 15s ago 12m 480M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283 running (2m) 19s ago 10m 99.4M - 19.0.0-1252-g44458cce 79957b372307 775a57400e80 2024-02-07T12:26:39.335 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (12m) 15s ago 12m 55.5M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (10m) 19s ago 10m 44.8M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (4m) 15s ago 4m 258M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (4m) 19s ago 4m 80.2M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (84s) 15s ago 10m 9499k - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (54s) 19s ago 9m 12.6M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (9m) 15s ago 9m 483M 1936M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (8m) 15s ago 8m 406M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (8m) 15s ago 8m 293M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:26:39.336 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (8m) 15s ago 8m 447M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:26:39.337 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (7m) 19s ago 7m 487M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:26:39.337 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (7m) 19s ago 7m 527M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:26:39.337 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (6m) 19s ago 6m 428M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:26:39.337 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (6m) 19s ago 6m 352M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:26:39.337 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (57s) 15s ago 9m 52.6M - 2.33.4 514e6a882f6e e735899d4840 2024-02-07T12:26:39.725 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:39 smithi174 ceph-mon[24580]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:39.725 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:39 smithi174 ceph-mon[24580]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.8 MiB/s wr, 675 op/s 2024-02-07T12:26:39.725 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:39 smithi174 ceph-mon[24580]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:39.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:39 smithi184 ceph-mon[27098]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:39.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:39 smithi184 ceph-mon[27098]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.8 MiB/s wr, 675 op/s 2024-02-07T12:26:39.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:39 smithi184 ceph-mon[27098]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:39.968 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:26:39.968 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:26:39.968 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:26:39.968 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:26:39.968 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:26:39.968 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:26:39.968 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:26:39.969 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:26:40.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 149843 12.21 MB/sec execute 147 sec latency 32.636 ms 2024-02-07T12:26:40.411 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:26:40.411 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:26:40.411 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:26:40.412 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:26:40.412 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:26:40.412 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:26:40.412 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:26:40.412 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "2/23 daemons upgraded", 2024-02-07T12:26:40.412 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "", 2024-02-07T12:26:40.412 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:26:40.412 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:26:40.745 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:40 smithi174 ceph-mon[24580]: from='client.14842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:40.745 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:40 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/2348499495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:26:40.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:40 smithi184 ceph-mon[27098]: from='client.14842 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:40.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:40 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/2348499495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:26:40.973 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:26:41.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 150122 12.15 MB/sec execute 148 sec latency 42.041 ms 2024-02-07T12:26:41.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:41 smithi174 ceph-mon[24580]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:41.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:41 smithi174 ceph-mon[24580]: pgmap v16: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.1 MiB/s wr, 879 op/s 2024-02-07T12:26:41.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:41 smithi174 ceph-mon[24580]: from='client.? 172.21.15.174:0/503741144' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:26:41.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:41 smithi184 ceph-mon[27098]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:26:41.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:41 smithi184 ceph-mon[27098]: pgmap v16: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.1 MiB/s wr, 879 op/s 2024-02-07T12:26:41.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:41 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/503741144' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:26:42.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 150336 12.10 MB/sec execute 149 sec latency 63.021 ms 2024-02-07T12:26:43.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 150651 12.10 MB/sec execute 150 sec latency 52.375 ms 2024-02-07T12:26:43.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:43 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:43.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:43 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:43.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:43 smithi174 ceph-mon[24580]: pgmap v17: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 824 op/s 2024-02-07T12:26:43.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:43 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:26:43.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:43 smithi174 ceph-mon[24580]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:26:43.545 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:43 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:43.789 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:43 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:43.789 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:43 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:43.789 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:43 smithi184 ceph-mon[27098]: pgmap v17: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 824 op/s 2024-02-07T12:26:43.789 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:43 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:26:43.790 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:43 smithi184 ceph-mon[27098]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:26:43.790 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:43 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:44.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 151225 12.11 MB/sec execute 151 sec latency 44.505 ms 2024-02-07T12:26:45.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 151756 12.11 MB/sec execute 152 sec latency 31.014 ms 2024-02-07T12:26:45.576 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:45 smithi174 ceph-mon[24580]: pgmap v18: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.9 MiB/s wr, 641 op/s 2024-02-07T12:26:45.576 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:45 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:45.576 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:45 smithi174 ceph-mon[24580]: Upgrade: Updating mgr.smithi184.prgadw 2024-02-07T12:26:45.576 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:45 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:26:45.576 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:45 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:26:45.576 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:45 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:45.576 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:45 smithi174 ceph-mon[24580]: Deploying daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:26:45.681 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:45 smithi184 ceph-mon[27098]: pgmap v18: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.9 MiB/s wr, 641 op/s 2024-02-07T12:26:45.681 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:45 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:26:45.681 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:45 smithi184 ceph-mon[27098]: Upgrade: Updating mgr.smithi184.prgadw 2024-02-07T12:26:45.681 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:45 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:26:45.681 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:45 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:26:45.682 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:45 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:26:45.682 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:45 smithi184 ceph-mon[27098]: Deploying daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:26:46.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 152295 12.15 MB/sec execute 153 sec latency 26.544 ms 2024-02-07T12:26:47.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 152766 12.18 MB/sec execute 154 sec latency 28.785 ms 2024-02-07T12:26:47.346 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:47 smithi174 ceph-mon[24580]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 886 op/s 2024-02-07T12:26:47.368 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:47 smithi184 ceph-mon[27098]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 886 op/s 2024-02-07T12:26:48.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 153172 12.19 MB/sec execute 155 sec latency 36.025 ms 2024-02-07T12:26:49.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 153493 12.13 MB/sec execute 156 sec latency 119.595 ms 2024-02-07T12:26:49.625 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:49 smithi174 ceph-mon[24580]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.5 MiB/s wr, 541 op/s 2024-02-07T12:26:49.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:49 smithi184 ceph-mon[27098]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.5 MiB/s wr, 541 op/s 2024-02-07T12:26:50.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 153740 12.07 MB/sec execute 157 sec latency 56.385 ms 2024-02-07T12:26:50.887 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:50 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:50.887 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:50 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:50.887 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:50 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:50.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:50 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:50.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:50 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:50.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:50 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:26:51.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 154056 12.06 MB/sec execute 158 sec latency 30.391 ms 2024-02-07T12:26:51.907 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:51 smithi174 ceph-mon[24580]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-02-07T12:26:51.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:51 smithi184 ceph-mon[27098]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-02-07T12:26:52.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 154330 12.03 MB/sec execute 159 sec latency 62.613 ms 2024-02-07T12:26:53.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 154594 12.00 MB/sec execute 160 sec latency 106.687 ms 2024-02-07T12:26:53.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:53 smithi174 ceph-mon[24580]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 578 op/s 2024-02-07T12:26:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:53 smithi184 ceph-mon[27098]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 578 op/s 2024-02-07T12:26:54.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 154893 11.96 MB/sec execute 161 sec latency 62.127 ms 2024-02-07T12:26:54.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:54 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:54.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:54 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:54.864 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:54 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:54.864 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:54 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:55.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 155209 11.91 MB/sec execute 162 sec latency 46.168 ms 2024-02-07T12:26:55.836 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:55 smithi174 ceph-mon[24580]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 485 op/s 2024-02-07T12:26:55.836 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:55 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:55.837 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:55 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:55.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:55 smithi184 ceph-mon[27098]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 485 op/s 2024-02-07T12:26:55.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:55 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:55.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:55 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:56.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 155415 11.89 MB/sec execute 163 sec latency 45.405 ms 2024-02-07T12:26:57.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 156010 11.96 MB/sec execute 164 sec latency 33.057 ms 2024-02-07T12:26:57.259 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:57 smithi184 ceph-mon[27098]: pgmap v24: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 658 op/s 2024-02-07T12:26:57.361 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:57 smithi174 ceph-mon[24580]: pgmap v24: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 658 op/s 2024-02-07T12:26:58.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 156515 11.99 MB/sec execute 165 sec latency 34.531 ms 2024-02-07T12:26:58.538 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:58 smithi184 ceph-mon[27098]: Standby manager daemon smithi184.prgadw restarted 2024-02-07T12:26:58.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:58 smithi184 ceph-mon[27098]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:26:58.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:58 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/3103005239' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:26:58.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:58 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/3103005239' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:26:58.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:58 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/3103005239' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:26:58.539 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:58 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/3103005239' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:26:58.630 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:58 smithi174 ceph-mon[24580]: Standby manager daemon smithi184.prgadw restarted 2024-02-07T12:26:58.630 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:58 smithi174 ceph-mon[24580]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:26:58.630 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:58 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/3103005239' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:26:58.630 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:58 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/3103005239' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:26:58.630 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:58 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/3103005239' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:26:58.630 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:58 smithi174 ceph-mon[24580]: from='mgr.? 172.21.15.184:0/3103005239' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:26:59.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 156965 11.97 MB/sec execute 166 sec latency 37.769 ms 2024-02-07T12:26:59.610 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:59 smithi174 ceph-mon[24580]: mgrmap e33: smithi174.whsozo(active, since 44s), standbys: smithi184.prgadw 2024-02-07T12:26:59.611 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:59 smithi174 ceph-mon[24580]: pgmap v25: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 413 op/s 2024-02-07T12:26:59.611 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:59 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:59.611 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:26:59 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:59.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:59 smithi184 ceph-mon[27098]: mgrmap e33: smithi174.whsozo(active, since 44s), standbys: smithi184.prgadw 2024-02-07T12:26:59.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:59 smithi184 ceph-mon[27098]: pgmap v25: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 413 op/s 2024-02-07T12:26:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:59 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:26:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:26:59 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.021 INFO:teuthology.orchestra.run.smithi174.stdout: 5 157369 11.93 MB/sec execute 167 sec latency 38.774 ms 2024-02-07T12:27:00.627 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.627 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.627 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:00.627 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:00.627 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.627 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: Upgrade: Setting container_image for all mgr 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.whsozo"}]: dispatch 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.whsozo"}]': finished 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi184.prgadw"}]: dispatch 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi184.prgadw"}]': finished 2024-02-07T12:27:00.628 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:00 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:27:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: Upgrade: Setting container_image for all mgr 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.whsozo"}]: dispatch 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.whsozo"}]': finished 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi184.prgadw"}]: dispatch 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi184.prgadw"}]': finished 2024-02-07T12:27:00.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:00 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:27:01.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 157770 11.95 MB/sec execute 168 sec latency 35.533 ms 2024-02-07T12:27:01.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:01 smithi174 ceph-mon[24580]: pgmap v26: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 439 op/s 2024-02-07T12:27:02.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 158296 11.95 MB/sec execute 169 sec latency 33.400 ms 2024-02-07T12:27:02.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:01 smithi184 ceph-mon[27098]: pgmap v26: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 439 op/s 2024-02-07T12:27:03.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 158742 11.92 MB/sec execute 170 sec latency 32.230 ms 2024-02-07T12:27:03.944 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:03 smithi174 ceph-mon[24580]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.6 MiB/s wr, 650 op/s 2024-02-07T12:27:03.945 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:03 smithi174 ceph-mon[24580]: Upgrade: Updating mon.smithi174 2024-02-07T12:27:03.945 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:03 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:03.945 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:03 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:27:03.945 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:03 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:27:03.945 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:03 smithi174 ceph-mon[24580]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:03.945 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:03 smithi174 ceph-mon[24580]: Deploying daemon mon.smithi174 on smithi174 2024-02-07T12:27:04.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 159096 11.92 MB/sec execute 171 sec latency 51.880 ms 2024-02-07T12:27:04.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:03 smithi184 ceph-mon[27098]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.6 MiB/s wr, 650 op/s 2024-02-07T12:27:04.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:03 smithi184 ceph-mon[27098]: Upgrade: Updating mon.smithi174 2024-02-07T12:27:04.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:03 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:04.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:03 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:27:04.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:03 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:27:04.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:03 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:04.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:03 smithi184 ceph-mon[27098]: Deploying daemon mon.smithi174 on smithi174 2024-02-07T12:27:05.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 159588 11.96 MB/sec execute 172 sec latency 41.043 ms 2024-02-07T12:27:05.968 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:05 smithi174 ceph-mon[24580]: pgmap v28: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 447 op/s 2024-02-07T12:27:05.968 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:05 smithi174 systemd[1]: Stopping Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:27:06.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 160035 11.99 MB/sec execute 173 sec latency 49.693 ms 2024-02-07T12:27:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:05 smithi184 ceph-mon[27098]: pgmap v28: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 447 op/s 2024-02-07T12:27:06.220 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:06 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[24576]: 2024-02-07T12:27:06.205+0000 7ff94894a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi174 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:27:06.220 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:06 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[24576]: 2024-02-07T12:27:06.205+0000 7ff94894a700 -1 mon.smithi174@0(leader) e2 *** Got Signal Terminated *** 2024-02-07T12:27:06.470 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:06 smithi174 podman[66053]: 2024-02-07 12:27:06.237827685 +0000 UTC m=+0.166449211 container died 812c56ca3a759b56c90acc06dbeca4eefe5b5765b36b7955403bfa2beb5f3a24 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, vcs-type=git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, ceph=True, GIT_BRANCH=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, build-date=2022-05-03T08:36:31.336870) 2024-02-07T12:27:06.978 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:06 smithi174 podman[66053]: 2024-02-07 12:27:06.782970384 +0000 UTC m=+0.711591902 container cleanup 812c56ca3a759b56c90acc06dbeca4eefe5b5765b36b7955403bfa2beb5f3a24 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-type=git, distribution-scope=public, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vendor=Red Hat, Inc., GIT_CLEAN=True) 2024-02-07T12:27:06.979 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:06 smithi174 bash[66053]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174 2024-02-07T12:27:07.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 160329 11.96 MB/sec execute 174 sec latency 36.210 ms 2024-02-07T12:27:07.238 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:07 smithi174 podman[66067]: 2024-02-07 12:27:07.150549665 +0000 UTC m=+0.908106382 container remove 812c56ca3a759b56c90acc06dbeca4eefe5b5765b36b7955403bfa2beb5f3a24 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.component=centos-stream-container, ceph=True, build-date=2022-05-03T08:36:31.336870, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, version=8, io.openshift.expose-services=, architecture=x86_64, distribution-scope=public, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-02-07T12:27:07.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:07 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service: Deactivated successfully. 2024-02-07T12:27:07.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:07 smithi174 systemd[1]: Stopped Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:27:07.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:07 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service: Consumed 16.713s CPU time. 2024-02-07T12:27:07.992 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:07 smithi174 systemd[1]: Starting Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:27:08.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 160575 11.90 MB/sec execute 175 sec latency 52.845 ms 2024-02-07T12:27:08.244 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 podman[66204]: 2024-02-07 12:27:07.992404462 +0000 UTC m=+0.024562299 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:27:08.245 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 podman[66204]: 2024-02-07 12:27:08.205858923 +0000 UTC m=+0.238016744 container create 245621856c3fd9c4bab9556341ce5f8f875efaa56ef8340f34573692b82f8f9a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_CLEAN=True, io.buildah.version=1.33.2, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-02-07T12:27:08.749 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 podman[66204]: 2024-02-07 12:27:08.592277735 +0000 UTC m=+0.624435563 container init 245621856c3fd9c4bab9556341ce5f8f875efaa56ef8340f34573692b82f8f9a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:27:08.749 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 podman[66204]: 2024-02-07 12:27:08.604088474 +0000 UTC m=+0.636246305 container start 245621856c3fd9c4bab9556341ce5f8f875efaa56ef8340f34573692b82f8f9a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, ceph=True, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev), process ceph-mon, pid 2 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: pidfile_write: ignore empty --pid-file 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: load: jerasure load: lrc 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: RocksDB version: 7.9.2 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Git sha 0 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Compile date 2024-02-05 19:41:51 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: DB SUMMARY 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: DB Session ID: ZV46NI4OX9E361CX02X9 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:27:08.750 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi174/store.db dir, Total Num: 1, files: 000030.sst 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi174/store.db: 000028.log size: 2609596 ; 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.flush_verify_memtable_count: 1 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.env: 0x564bb6adaae0 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.fs: PosixFileSystem 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.info_log: 0x564bb7f87760 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.statistics: (nil) 2024-02-07T12:27:08.751 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.use_fsync: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:27:08.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.db_log_dir: 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.wal_dir: 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.write_buffer_manager: 0x564bb7f5b400 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:27:08.753 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.unordered_write: 0 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.row_cache: None 2024-02-07T12:27:08.754 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.wal_filter: None 2024-02-07T12:27:08.755 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:27:08.755 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.wal_compression: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:27:08.756 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.enforce_single_del_contracts: true 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_open_files: -1 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:27:08.757 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Compression algorithms supported: 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: kZSTD supported: 0 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: DMutex implementation: pthread_mutex_t 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi174/store.db/MANIFEST-000015 2024-02-07T12:27:08.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.merge_operator: 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_filter: None 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564bb7f87880) 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_type: 0 2024-02-07T12:27:08.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_block_index_type: 0 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_shortening: 1 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: checksum: 4 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: no_block_cache: 0 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache: 0x564bb7f8b1f0 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_options: 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: capacity : 536870912 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: num_shard_bits : 4 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: strict_capacity_limit : 0 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_compressed: (nil) 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: persistent_cache: (nil) 2024-02-07T12:27:08.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_size: 4096 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_size_deviation: 10 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_restart_interval: 16 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_block_restart_interval: 1 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: metadata_block_size: 4096 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: partition_filters: 0 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: use_delta_encoding: 1 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: filter_policy: bloomfilter 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: whole_key_filtering: 1 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: verify_compression: 0 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: format_version: 5 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: enable_index_compression: 1 2024-02-07T12:27:08.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_align: 0 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_auto_readahead_size: 262144 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: prepopulate_block_cache: 0 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: initial_auto_readahead_size: 8192 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: num_file_reads_for_auto_readahead: 2 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression: NoCompression 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.num_levels: 7 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:27:08.762 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:27:08.763 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:27:08.764 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.arena_block_size: 1048576 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:27:08.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:27:08.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.ttl: 2592000 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-02-07T12:27:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.blob_file_starting_level: 0 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi174/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11825, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1fd786af-80fc-4375-bba5-16f10eb5cbca 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308828656560, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308828677723, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2391606, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11826, "largest_seqno": 12941, "table_properties": {"data_size": 2386102, "index_size": 3269, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10870, "raw_average_key_size": 22, "raw_value_size": 2375795, "raw_average_value_size": 5012, "num_data_blocks": 149, "num_entries": 474, "num_filter_entries": 474, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1707308828, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1fd786af-80fc-4375-bba5-16f10eb5cbca", "db_session_id": "ZV46NI4OX9E361CX02X9", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308828677881, "job": 1, "event": "recovery_finished"} 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:27:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi174/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564bb807a000 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: DB pointer 0x564bb8066000 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** DB Stats ** 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-02-07T12:27:08.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: L0 1/0 2.28 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 236.0 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: L6 1/0 6.48 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Sum 2/0 8.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 236.0 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 236.0 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 236.0 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:27:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative compaction: 0.00 GB write, 78.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval compaction: 0.00 GB write, 78.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Block cache BinnedLRUCache@0x564bb7f8b1f0#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.8e-05 secs_since: 0 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:27:08.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: starting mon.smithi174 rank 0 at public addrs [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] at bind addrs [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi174 fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:27:08.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: mon.smithi174@-1(???) e2 preinit fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:27:08.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: mon.smithi174@-1(???).mds e7 new map 2024-02-07T12:27:08.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: mon.smithi174@-1(???).mds e7 print_map 2024-02-07T12:27:08.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: e7 2024-02-07T12:27:08.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:27:08.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:27:08.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: legacy client fscid: 1 2024-02-07T12:27:08.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Filesystem 'foofs' (1) 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: fs_name foofs 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: epoch 5 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: created 2024-02-07T12:21:34.220116+0000 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: modified 2024-02-07T12:21:40.240137+0000 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: tableserver 0 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: root 0 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: session_timeout 60 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: session_autoclose 300 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_file_size 1099511627776 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_xattr_size 65536 2024-02-07T12:27:08.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: required_client_features {} 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: last_failure 0 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: last_failure_osd_epoch 0 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_mds 1 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: in 0 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: up {0=14502} 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: failed 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: damaged 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: stopped 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_pools [3] 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: metadata_pool 2 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: inline_data disabled 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: balancer 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: bal_rank_mask -1 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: standby_count_wanted 1 2024-02-07T12:27:08.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: [mds.foofs.smithi174.vkrzkp{0:14502} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.174:6834/1377814839,v1:172.21.15.174:6835/1377814839] compat {c=[1],r=[1],i=[7ff]}] 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Standby daemons: 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: [mds.foofs.smithi184.alpixp{-1:24269} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] compat {c=[1],r=[1],i=[7ff]}] 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: mon.smithi174@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: mon.smithi174@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: mon.smithi174@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: mon.smithi174@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:27:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 ceph-mon[66222]: mon.smithi174@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-02-07T12:27:09.012 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 bash[66204]: 245621856c3fd9c4bab9556341ce5f8f875efaa56ef8340f34573692b82f8f9a 2024-02-07T12:27:09.012 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:08 smithi174 systemd[1]: Started Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:27:09.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 160799 11.85 MB/sec execute 176 sec latency 72.478 ms 2024-02-07T12:27:09.533 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: pgmap v29: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.4 MiB/s wr, 715 op/s 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: pgmap v30: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.9 MiB/s wr, 542 op/s 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: mon.smithi174 calling monitor election 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: mon.smithi174 is new leader, mons smithi174,smithi184 in quorum (ranks 0,1) 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: monmap e2: 2 mons at {smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0],smithi184=[v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: osdmap e72: 8 total, 8 up, 8 in 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: mgrmap e33: smithi174.whsozo(active, since 55s), standbys: smithi184.prgadw 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: from='mgr.14782 ' entity='' 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: mgrmap e34: smithi174.whsozo(active, since 55s), standbys: smithi184.prgadw 2024-02-07T12:27:09.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:09 smithi174 ceph-mon[66222]: overall HEALTH_OK 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: from='mgr.14782 172.21.15.174:0/597486571' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: pgmap v29: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.4 MiB/s wr, 715 op/s 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: pgmap v30: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.9 MiB/s wr, 542 op/s 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: mon.smithi174 calling monitor election 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: mon.smithi174 is new leader, mons smithi174,smithi184 in quorum (ranks 0,1) 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: monmap e2: 2 mons at {smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0],smithi184=[v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: osdmap e72: 8 total, 8 up, 8 in 2024-02-07T12:27:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: mgrmap e33: smithi174.whsozo(active, since 55s), standbys: smithi184.prgadw 2024-02-07T12:27:09.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: from='mgr.14782 ' entity='' 2024-02-07T12:27:09.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: mgrmap e34: smithi174.whsozo(active, since 55s), standbys: smithi184.prgadw 2024-02-07T12:27:09.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:09 smithi184 ceph-mon[27098]: overall HEALTH_OK 2024-02-07T12:27:10.022 INFO:teuthology.orchestra.run.smithi174.stdout: 5 161193 11.86 MB/sec execute 177 sec latency 42.024 ms 2024-02-07T12:27:11.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 161825 11.88 MB/sec execute 178 sec latency 31.107 ms 2024-02-07T12:27:11.552 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:11 smithi174 ceph-mon[66222]: mgrmap e35: smithi174.whsozo(active, since 56s), standbys: smithi184.prgadw 2024-02-07T12:27:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:11 smithi184 ceph-mon[27098]: mgrmap e35: smithi174.whsozo(active, since 56s), standbys: smithi184.prgadw 2024-02-07T12:27:12.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 162309 11.85 MB/sec execute 179 sec latency 34.805 ms 2024-02-07T12:27:13.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 162982 11.92 MB/sec execute 180 sec latency 24.657 ms 2024-02-07T12:27:14.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 163532 11.98 MB/sec execute 181 sec latency 25.759 ms 2024-02-07T12:27:15.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 163923 11.96 MB/sec execute 182 sec latency 43.685 ms 2024-02-07T12:27:16.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 164298 11.91 MB/sec execute 183 sec latency 41.028 ms 2024-02-07T12:27:17.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 164601 11.89 MB/sec execute 184 sec latency 41.216 ms 2024-02-07T12:27:18.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 164927 11.89 MB/sec execute 185 sec latency 38.897 ms 2024-02-07T12:27:18.148 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:18 smithi174 ceph-mon[66222]: Standby manager daemon smithi184.prgadw restarted 2024-02-07T12:27:18.148 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:18 smithi174 ceph-mon[66222]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:27:18.148 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:18 smithi174 ceph-mon[66222]: from='mgr.? 172.21.15.184:0/374935694' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:27:18.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:18 smithi174 ceph-mon[66222]: from='mgr.? 172.21.15.184:0/374935694' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:27:18.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:18 smithi174 ceph-mon[66222]: from='mgr.? 172.21.15.184:0/374935694' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:27:18.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:18 smithi174 ceph-mon[66222]: from='mgr.? 172.21.15.184:0/374935694' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:27:18.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:18 smithi184 ceph-mon[27098]: Standby manager daemon smithi184.prgadw restarted 2024-02-07T12:27:18.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:18 smithi184 ceph-mon[27098]: Standby manager daemon smithi184.prgadw started 2024-02-07T12:27:18.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:18 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/374935694' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/crt"}]: dispatch 2024-02-07T12:27:18.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:18 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/374935694' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-02-07T12:27:18.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:18 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/374935694' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi184.prgadw/key"}]: dispatch 2024-02-07T12:27:18.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:18 smithi184 ceph-mon[27098]: from='mgr.? 172.21.15.184:0/374935694' entity='mgr.smithi184.prgadw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-02-07T12:27:19.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 165344 11.88 MB/sec execute 186 sec latency 55.635 ms 2024-02-07T12:27:19.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:19 smithi174 ceph-mon[66222]: mgrmap e36: smithi174.whsozo(active, since 64s), standbys: smithi184.prgadw 2024-02-07T12:27:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:19 smithi184 ceph-mon[27098]: mgrmap e36: smithi174.whsozo(active, since 64s), standbys: smithi184.prgadw 2024-02-07T12:27:20.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 165675 11.84 MB/sec execute 187 sec latency 66.821 ms 2024-02-07T12:27:20.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: Active manager daemon smithi174.whsozo restarted 2024-02-07T12:27:20.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: Activating manager daemon smithi174.whsozo 2024-02-07T12:27:20.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: osdmap e73: 8 total, 8 up, 8 in 2024-02-07T12:27:20.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: mgrmap e37: smithi174.whsozo(active, starting, since 0.00714501s), standbys: smithi184.prgadw 2024-02-07T12:27:20.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:27:20.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:27:20.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi184.prgadw", "id": "smithi184.prgadw"}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:27:20.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:19 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184[27092]: 2024-02-07T12:27:19.961+0000 7f132cdaf700 -1 mon.smithi184@1(peon).osd e73 update_from_paxos full map CRC mismatch, resetting to canonical 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: Active manager daemon smithi174.whsozo restarted 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: Activating manager daemon smithi174.whsozo 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: osdmap e73: 8 total, 8 up, 8 in 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: mgrmap e37: smithi174.whsozo(active, starting, since 0.00714501s), standbys: smithi184.prgadw 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:27:20.291 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi174.whsozo", "id": "smithi174.whsozo"}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr metadata", "who": "smithi184.prgadw", "id": "smithi184.prgadw"}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-02-07T12:27:20.292 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:20 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-02-07T12:27:21.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 165918 11.81 MB/sec execute 188 sec latency 44.457 ms 2024-02-07T12:27:21.043 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: failed to encode map e73 with expected crc 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: failed to encode map e73 with expected crc 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: failed to encode map e73 with expected crc 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:21.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:21 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: failed to encode map e73 with expected crc 2024-02-07T12:27:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: failed to encode map e73 with expected crc 2024-02-07T12:27:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: Manager daemon smithi174.whsozo is now available 2024-02-07T12:27:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: failed to encode map e73 with expected crc 2024-02-07T12:27:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:27:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/mirror_snapshot_schedule"}]: dispatch 2024-02-07T12:27:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:21 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.whsozo/trash_purge_schedule"}]: dispatch 2024-02-07T12:27:21.918 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:27:21.918 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (2m) 22s ago 11m 23.8M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (11m) 22s ago 11m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (10m) 28s ago 10m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (2m) 22s ago 11m 45.6M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (5m) 22s ago 5m 38.0M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (5m) 28s ago 5m 12.5M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (86s) 22s ago 12m 493M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (33s) 28s ago 10m 64.9M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (12m) 22s ago 13m 54.8M 2048M 17.2.0 e1d6a67b021e 812c56ca3a75 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (10m) 28s ago 10m 43.8M 2048M 17.2.0 e1d6a67b021e 11df1ddcd533 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (5m) 22s ago 5m 262M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (5m) 28s ago 5m 81.8M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (2m) 22s ago 11m 12.9M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (97s) 28s ago 10m 13.8M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (10m) 22s ago 10m 525M 1936M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:27:21.919 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (9m) 22s ago 9m 458M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:27:21.920 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (9m) 22s ago 9m 339M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:27:21.920 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (8m) 22s ago 8m 494M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:27:21.920 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (8m) 28s ago 8m 504M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:27:21.920 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (8m) 28s ago 8m 321M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:27:21.920 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (7m) 28s ago 7m 491M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:27:21.920 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (7m) 28s ago 7m 386M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:27:21.920 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (40s) 22s ago 10m 52.8M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:27:22.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 166212 11.80 MB/sec execute 189 sec latency 51.661 ms 2024-02-07T12:27:22.240 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:22 smithi174 ceph-mon[66222]: [07/Feb/2024:12:27:20] ENGINE Bus STARTING 2024-02-07T12:27:22.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:22 smithi174 ceph-mon[66222]: [07/Feb/2024:12:27:20] ENGINE Serving on https://172.21.15.174:7150 2024-02-07T12:27:22.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:22 smithi174 ceph-mon[66222]: [07/Feb/2024:12:27:20] ENGINE Serving on http://172.21.15.174:8765 2024-02-07T12:27:22.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:22 smithi174 ceph-mon[66222]: [07/Feb/2024:12:27:20] ENGINE Bus STARTED 2024-02-07T12:27:22.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:22 smithi174 ceph-mon[66222]: mgrmap e38: smithi174.whsozo(active, since 1.05974s), standbys: smithi184.prgadw 2024-02-07T12:27:22.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:22 smithi174 ceph-mon[66222]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:22.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:22 smithi174 ceph-mon[66222]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:27:22.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:22 smithi174 ceph-mon[66222]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:22.412 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:27:22.412 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:27:22.412 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-02-07T12:27:22.412 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:27:22.412 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:27:22.413 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-02-07T12:27:22.414 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:27:22.414 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:27:22.414 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:27:22.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:22 smithi184 ceph-mon[27098]: [07/Feb/2024:12:27:20] ENGINE Bus STARTING 2024-02-07T12:27:22.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:22 smithi184 ceph-mon[27098]: [07/Feb/2024:12:27:20] ENGINE Serving on https://172.21.15.174:7150 2024-02-07T12:27:22.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:22 smithi184 ceph-mon[27098]: [07/Feb/2024:12:27:20] ENGINE Serving on http://172.21.15.174:8765 2024-02-07T12:27:22.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:22 smithi184 ceph-mon[27098]: [07/Feb/2024:12:27:20] ENGINE Bus STARTED 2024-02-07T12:27:22.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:22 smithi184 ceph-mon[27098]: mgrmap e38: smithi174.whsozo(active, since 1.05974s), standbys: smithi184.prgadw 2024-02-07T12:27:22.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:22 smithi184 ceph-mon[27098]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:22.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:22 smithi184 ceph-mon[27098]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:27:22.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:22 smithi184 ceph-mon[27098]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:22.992 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:27:22.992 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:27:22.992 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:27:22.993 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:27:22.993 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:27:22.993 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:27:22.993 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:27:22.993 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "2/23 daemons upgraded", 2024-02-07T12:27:22.993 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "", 2024-02-07T12:27:22.993 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:27:22.993 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:27:23.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 166550 11.81 MB/sec execute 190 sec latency 38.166 ms 2024-02-07T12:27:23.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:23 smithi174 ceph-mon[66222]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:23.276 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:23 smithi174 ceph-mon[66222]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:27:23.276 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:23 smithi174 ceph-mon[66222]: mgrmap e39: smithi174.whsozo(active, since 2s), standbys: smithi184.prgadw 2024-02-07T12:27:23.276 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:23 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3477354243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:23.366 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:23 smithi184 ceph-mon[27098]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:23.366 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:23 smithi184 ceph-mon[27098]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:27:23.366 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:23 smithi184 ceph-mon[27098]: mgrmap e39: smithi174.whsozo(active, since 2s), standbys: smithi184.prgadw 2024-02-07T12:27:23.366 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:23 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/3477354243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:23.470 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:27:24.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 166912 11.82 MB/sec execute 191 sec latency 41.907 ms 2024-02-07T12:27:24.060 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:24 smithi174 ceph-mon[66222]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:24.060 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:24 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/645423451' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:27:24.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:24 smithi184 ceph-mon[27098]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:24.391 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:24 smithi184 ceph-mon[27098]: from='client.? 172.21.15.174:0/645423451' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:27:25.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 167307 11.83 MB/sec execute 192 sec latency 34.846 ms 2024-02-07T12:27:25.074 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:25 smithi174 ceph-mon[66222]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:27:25.075 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:25 smithi174 ceph-mon[66222]: mgrmap e40: smithi174.whsozo(active, since 4s), standbys: smithi184.prgadw 2024-02-07T12:27:25.075 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:25 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:25.075 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:25 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:25 smithi184 ceph-mon[27098]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:27:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:25 smithi184 ceph-mon[27098]: mgrmap e40: smithi174.whsozo(active, since 4s), standbys: smithi184.prgadw 2024-02-07T12:27:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:25 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:25 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:26.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 167667 11.78 MB/sec execute 193 sec latency 37.956 ms 2024-02-07T12:27:27.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 168073 11.77 MB/sec execute 194 sec latency 38.428 ms 2024-02-07T12:27:27.081 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:27 smithi174 ceph-mon[66222]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:27:27.082 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:27 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:27.082 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:27 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:27.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:27 smithi184 ceph-mon[27098]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-02-07T12:27:27.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:27 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:27.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:27 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:28.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 168518 11.77 MB/sec execute 195 sec latency 46.423 ms 2024-02-07T12:27:29.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 169068 11.78 MB/sec execute 196 sec latency 34.851 ms 2024-02-07T12:27:29.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:29 smithi174 ceph-mon[66222]: pgmap v7: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 4.2 MiB/s wr, 373 op/s 2024-02-07T12:27:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:29 smithi184 ceph-mon[27098]: pgmap v7: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 4.2 MiB/s wr, 373 op/s 2024-02-07T12:27:30.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 169452 11.75 MB/sec execute 197 sec latency 38.350 ms 2024-02-07T12:27:30.845 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:30 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:30.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:30 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:30.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:30 smithi174 ceph-mon[66222]: pgmap v8: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 MiB/s wr, 289 op/s 2024-02-07T12:27:30.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:30 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:30.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:30 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:31.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 169914 11.78 MB/sec execute 198 sec latency 38.176 ms 2024-02-07T12:27:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:30 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:30 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:30 smithi184 ceph-mon[27098]: pgmap v8: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 MiB/s wr, 289 op/s 2024-02-07T12:27:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:30 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:30 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:32.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 170363 11.81 MB/sec execute 199 sec latency 39.917 ms 2024-02-07T12:27:32.378 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:32 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:32.378 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:32 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:32.379 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:32.379 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:32.379 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:32 smithi174 ceph-mon[66222]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:27:32.379 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:32 smithi174 ceph-mon[66222]: Updating smithi184:/etc/ceph/ceph.conf 2024-02-07T12:27:32.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:32 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:32.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:32 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:32.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:32 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:32.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:32 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:32.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:32 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.conf 2024-02-07T12:27:32.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:32 smithi184 ceph-mon[27098]: Updating smithi184:/etc/ceph/ceph.conf 2024-02-07T12:27:33.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 170681 11.82 MB/sec execute 200 sec latency 55.746 ms 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: Updating smithi184:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: pgmap v9: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.5 MiB/s wr, 546 op/s 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:33.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:33.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:33 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:27:33.551 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:27:33.552 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.conf 2024-02-07T12:27:33.552 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: Updating smithi184:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:27:33.552 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-02-07T12:27:33.552 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: pgmap v9: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.5 MiB/s wr, 546 op/s 2024-02-07T12:27:33.552 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: Updating smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:27:33.552 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: Updating smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/config/ceph.client.admin.keyring 2024-02-07T12:27:33.552 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.553 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.553 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.553 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.553 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.553 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:33.553 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:33.553 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:33.553 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:33 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-02-07T12:27:34.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 170953 11.78 MB/sec execute 201 sec latency 51.734 ms 2024-02-07T12:27:35.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 171217 11.73 MB/sec execute 202 sec latency 39.842 ms 2024-02-07T12:27:35.430 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:35 smithi174 ceph-mon[66222]: pgmap v10: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-02-07T12:27:35.431 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:35 smithi174 ceph-mon[66222]: Upgrade: Updating mon.smithi184 2024-02-07T12:27:35.431 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:35 smithi174 ceph-mon[66222]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:35.431 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:27:35.431 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:27:35.431 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:35.431 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:35 smithi174 ceph-mon[66222]: Deploying daemon mon.smithi184 on smithi184 2024-02-07T12:27:35.431 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:27:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:35 smithi184 ceph-mon[27098]: pgmap v10: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-02-07T12:27:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:35 smithi184 ceph-mon[27098]: Upgrade: Updating mon.smithi184 2024-02-07T12:27:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:35 smithi184 ceph-mon[27098]: from='mgr.34102 ' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:35 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:27:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:35 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:27:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:35 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:35 smithi184 ceph-mon[27098]: Deploying daemon mon.smithi184 on smithi184 2024-02-07T12:27:35.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:35 smithi184 ceph-mon[27098]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:27:36.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 171514 11.71 MB/sec execute 203 sec latency 70.887 ms 2024-02-07T12:27:36.640 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:36 smithi184 systemd[1]: Stopping Ceph mon.smithi184 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:27:36.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184[27092]: 2024-02-07T12:27:36.635+0000 7f13325ba700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi184 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:27:36.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184[27092]: 2024-02-07T12:27:36.635+0000 7f13325ba700 -1 mon.smithi184@1(peon) e2 *** Got Signal Terminated *** 2024-02-07T12:27:36.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:36 smithi184 podman[52901]: 2024-02-07 12:27:36.6779269 +0000 UTC m=+0.188393798 container died 11df1ddcd5336ed5ad94cf2a17f98d31e211fb4d5e91f05ec1a4d4231f6b97a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, release=754, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.openshift.expose-services=, distribution-scope=public, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-02-07T12:27:37.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 171814 11.69 MB/sec execute 204 sec latency 41.242 ms 2024-02-07T12:27:37.340 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 podman[52901]: 2024-02-07 12:27:37.0767144 +0000 UTC m=+0.587181299 container cleanup 11df1ddcd5336ed5ad94cf2a17f98d31e211fb4d5e91f05ec1a4d4231f6b97a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, com.redhat.component=centos-stream-container, release=754, io.openshift.expose-services=, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, RELEASE=HEAD, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-02-07T12:27:37.340 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 bash[52901]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184 2024-02-07T12:27:37.691 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 podman[52913]: 2024-02-07 12:27:37.338410411 +0000 UTC m=+0.655821054 container remove 11df1ddcd5336ed5ad94cf2a17f98d31e211fb4d5e91f05ec1a4d4231f6b97a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , name=centos-stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=) 2024-02-07T12:27:37.692 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi184.service: Deactivated successfully. 2024-02-07T12:27:37.692 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 systemd[1]: Stopped Ceph mon.smithi184 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:27:37.692 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi184.service: Consumed 7.152s CPU time. 2024-02-07T12:27:37.692 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 systemd[1]: Starting Ceph mon.smithi184 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:27:37.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 podman[53042]: 2024-02-07 12:27:37.818695733 +0000 UTC m=+0.021305908 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:27:37.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:37 smithi184 podman[53042]: 2024-02-07 12:27:37.93152843 +0000 UTC m=+0.134138597 container create 40219d987787438b07b700b3d017e3e329fbf44fe8cce7ba344b8d189e4a1451 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-02-07T12:27:38.023 INFO:teuthology.orchestra.run.smithi174.stdout: 5 172135 11.68 MB/sec execute 205 sec latency 63.148 ms 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 podman[53042]: 2024-02-07 12:27:38.176330241 +0000 UTC m=+0.378940409 container init 40219d987787438b07b700b3d017e3e329fbf44fe8cce7ba344b8d189e4a1451 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 podman[53042]: 2024-02-07 12:27:38.180202025 +0000 UTC m=+0.382812198 container start 40219d987787438b07b700b3d017e3e329fbf44fe8cce7ba344b8d189e4a1451 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_BRANCH=HEAD) 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: set uid:gid to 167:167 (ceph:ceph) 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev), process ceph-mon, pid 2 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: pidfile_write: ignore empty --pid-file 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: load: jerasure load: lrc 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: RocksDB version: 7.9.2 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Git sha 0 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Compile date 2024-02-05 19:41:51 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: DB SUMMARY 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: DB Session ID: X5XCQA82QEHGD1QHHMV7 2024-02-07T12:27:38.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: CURRENT file: CURRENT 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: IDENTITY file: IDENTITY 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi184/store.db dir, Total Num: 1, files: 000024.sst 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi184/store.db: 000022.log size: 7113386 ; 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.error_if_exists: 0 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.create_if_missing: 0 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.paranoid_checks: 1 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.flush_verify_memtable_count: 1 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.env: 0x55730f0dbae0 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.fs: PosixFileSystem 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.info_log: 0x557310ca1760 2024-02-07T12:27:38.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_file_opening_threads: 16 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.statistics: (nil) 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.use_fsync: 0 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_log_file_size: 0 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.log_file_time_to_roll: 0 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.keep_log_file_num: 1000 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.recycle_log_file_num: 0 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.allow_fallocate: 1 2024-02-07T12:27:38.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.allow_mmap_reads: 0 2024-02-07T12:27:38.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.allow_mmap_writes: 0 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.use_direct_reads: 0 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.create_missing_column_families: 0 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.db_log_dir: 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.wal_dir: 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.table_cache_numshardbits: 6 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.WAL_ttl_seconds: 0 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.WAL_size_limit_MB: 0 2024-02-07T12:27:38.455 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.is_fd_close_on_exec: 1 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.advise_random_on_open: 1 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.db_write_buffer_size: 0 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.write_buffer_manager: 0x557310c75400 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.use_adaptive_mutex: 0 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.rate_limiter: (nil) 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.wal_recovery_mode: 2 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.enable_thread_tracking: 0 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.enable_pipelined_write: 0 2024-02-07T12:27:38.456 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.unordered_write: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.row_cache: None 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.wal_filter: None 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.allow_ingest_behind: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.two_write_queues: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.manual_wal_flush: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.wal_compression: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.atomic_flush: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.persist_stats_to_disk: 0 2024-02-07T12:27:38.457 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.write_dbid_to_manifest: 0 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.log_readahead_size: 0 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.best_efforts_recovery: 0 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.allow_data_in_errors: 0 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.db_host_id: __hostname__ 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.enforce_single_del_contracts: true 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_background_jobs: 2 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_background_compactions: -1 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_subcompactions: 1 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.delayed_write_rate : 16777216 2024-02-07T12:27:38.458 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_total_wal_size: 0 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.stats_dump_period_sec: 600 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.stats_persist_period_sec: 600 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_open_files: -1 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bytes_per_sync: 0 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.wal_bytes_per_sync: 0 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.strict_bytes_per_sync: 0 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_readahead_size: 0 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_background_flushes: -1 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Compression algorithms supported: 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: kZSTD supported: 0 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: kXpressCompression supported: 0 2024-02-07T12:27:38.459 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: kLZ4HCCompression supported: 1 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: kZlibCompression supported: 1 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: kSnappyCompression supported: 1 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: kLZ4Compression supported: 1 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: kBZip2Compression supported: 0 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Fast CRC32 supported: Supported on x86 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: DMutex implementation: pthread_mutex_t 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi184/store.db/MANIFEST-000009 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.merge_operator: 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_filter: None 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_filter_factory: None 2024-02-07T12:27:38.460 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.sst_partitioner_factory: None 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.memtable_factory: SkipListFactory 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.table_factory: BlockBasedTable 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557310ca1880) 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: cache_index_and_filter_blocks: 1 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: pin_top_level_index_and_filter: 1 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: index_type: 0 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: data_block_index_type: 0 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: index_shortening: 1 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: data_block_hash_table_util_ratio: 0.750000 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: checksum: 4 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: no_block_cache: 0 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_cache: 0x557310ca5090 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_cache_name: BinnedLRUCache 2024-02-07T12:27:38.461 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_cache_options: 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: capacity : 536870912 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: num_shard_bits : 4 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: strict_capacity_limit : 0 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: high_pri_pool_ratio: 0.000 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_cache_compressed: (nil) 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: persistent_cache: (nil) 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_size: 4096 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_size_deviation: 10 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_restart_interval: 16 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: index_block_restart_interval: 1 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: metadata_block_size: 4096 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: partition_filters: 0 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: use_delta_encoding: 1 2024-02-07T12:27:38.462 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: filter_policy: bloomfilter 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: whole_key_filtering: 1 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: verify_compression: 0 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: read_amp_bytes_per_bit: 0 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: format_version: 5 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: enable_index_compression: 1 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: block_align: 0 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: max_auto_readahead_size: 262144 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: prepopulate_block_cache: 0 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: initial_auto_readahead_size: 8192 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: num_file_reads_for_auto_readahead: 2 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.write_buffer_size: 33554432 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_write_buffer_number: 2 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression: NoCompression 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression: Disabled 2024-02-07T12:27:38.463 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.prefix_extractor: nullptr 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.num_levels: 7 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:27:38.464 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.window_bits: -14 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.level: 32767 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.strategy: 0 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.enabled: false 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-02-07T12:27:38.465 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.target_file_size_base: 67108864 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.target_file_size_multiplier: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-02-07T12:27:38.466 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.arena_block_size: 1048576 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.disable_auto_compactions: 0 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-02-07T12:27:38.467 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.table_properties_collectors: 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.inplace_update_support: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.inplace_update_num_locks: 10000 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.memtable_huge_page_size: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.bloom_locality: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.max_successive_merges: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.optimize_filters_for_hits: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.paranoid_file_checks: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.force_consistency_checks: 1 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.report_bg_io_stats: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.ttl: 2592000 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.periodic_compaction_seconds: 0 2024-02-07T12:27:38.468 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.enable_blob_files: false 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.min_blob_size: 0 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.blob_file_size: 268435456 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.blob_compression_type: NoCompression 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.enable_blob_garbage_collection: false 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.blob_file_starting_level: 0 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi184/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12203, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-02-07T12:27:38.469 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 71eed56b-6432-4f3e-ad31-c55ca9a9dec9 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308858214118, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308858234322, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4317206, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12208, "largest_seqno": 14027, "table_properties": {"data_size": 4309495, "index_size": 4900, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1797, "raw_key_size": 16035, "raw_average_key_size": 22, "raw_value_size": 4294428, "raw_average_value_size": 6152, "num_data_blocks": 223, "num_entries": 698, "num_filter_entries": 698, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1707308858, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "71eed56b-6432-4f3e-ad31-c55ca9a9dec9", "db_session_id": "X5XCQA82QEHGD1QHHMV7", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: EVENT_LOG_v1 {"time_micros": 1707308858234440, "job": 1, "event": "recovery_finished"} 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi184/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557310d94000 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: DB pointer 0x557310d80000 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** DB Stats ** 2024-02-07T12:27:38.470 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** Compaction Stats [default] ** 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: L0 1/0 4.12 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 470.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: L6 1/0 6.48 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-02-07T12:27:38.471 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Sum 2/0 10.60 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 470.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 470.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** Compaction Stats [default] ** 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 470.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-02-07T12:27:38.472 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: AddFile(Keys): cumulative 0, interval 0 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Cumulative compaction: 0.00 GB write, 177.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Interval compaction: 0.00 GB write, 177.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Block cache BinnedLRUCache@0x557310ca5090#2 capacity: 512.00 MB usage: 7.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: ** File Read Latency Histogram By Level [default] ** 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: starting mon.smithi184 rank 1 at public addrs [v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0] at bind addrs [v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi184 fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: mon.smithi184@-1(???) e2 preinit fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: mon.smithi184@-1(???).mds e7 new map 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: mon.smithi184@-1(???).mds e7 print_map 2024-02-07T12:27:38.473 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: e7 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: legacy client fscid: 1 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Filesystem 'foofs' (1) 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: fs_name foofs 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: epoch 5 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: created 2024-02-07T12:21:34.220116+0000 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: modified 2024-02-07T12:21:40.240137+0000 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: tableserver 0 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: root 0 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: session_timeout 60 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: session_autoclose 300 2024-02-07T12:27:38.474 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: max_file_size 1099511627776 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: max_xattr_size 65536 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: required_client_features {} 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: last_failure 0 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: last_failure_osd_epoch 0 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: max_mds 1 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: in 0 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: up {0=14502} 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: failed 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: damaged 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: stopped 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: data_pools [3] 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: metadata_pool 2 2024-02-07T12:27:38.475 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: inline_data disabled 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: balancer 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: bal_rank_mask -1 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: standby_count_wanted 1 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: [mds.foofs.smithi174.vkrzkp{0:14502} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.174:6834/1377814839,v1:172.21.15.174:6835/1377814839] compat {c=[1],r=[1],i=[7ff]}] 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: Standby daemons: 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout: [mds.foofs.smithi184.alpixp{-1:24269} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] compat {c=[1],r=[1],i=[7ff]}] 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: mon.smithi184@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: mon.smithi184@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: mon.smithi184@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: mon.smithi184@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: mon.smithi184@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-02-07T12:27:38.476 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 bash[53042]: 40219d987787438b07b700b3d017e3e329fbf44fe8cce7ba344b8d189e4a1451 2024-02-07T12:27:38.477 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 systemd[1]: Started Ceph mon.smithi184 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:27:38.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:38 smithi174 ceph-mon[66222]: pgmap v11: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-02-07T12:27:38.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:38 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:27:38.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:38 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:27:39.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 172507 11.67 MB/sec execute 206 sec latency 56.636 ms 2024-02-07T12:27:39.146 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: pgmap v11: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-02-07T12:27:39.146 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:27:39.146 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:38 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:27:39.814 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: overall HEALTH_OK 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: mon.smithi184 calling monitor election 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: mon.smithi174 calling monitor election 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: mon.smithi174 is new leader, mons smithi174,smithi184 in quorum (ranks 0,1) 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: monmap e3: 2 mons at {smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0],smithi184=[v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: osdmap e73: 8 total, 8 up, 8 in 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: mgrmap e40: smithi174.whsozo(active, since 18s), standbys: smithi184.prgadw 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: overall HEALTH_OK 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:39.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:39.816 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:39 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:40.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 172781 11.62 MB/sec execute 207 sec latency 123.078 ms 2024-02-07T12:27:40.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: overall HEALTH_OK 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mon metadata", "id": "smithi184"}]: dispatch 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: mon.smithi184 calling monitor election 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: mon.smithi174 calling monitor election 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: mon.smithi174 is new leader, mons smithi174,smithi184 in quorum (ranks 0,1) 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: monmap e3: 2 mons at {smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0],smithi184=[v2:172.21.15.184:3300/0,v1:172.21.15.184:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: osdmap e73: 8 total, 8 up, 8 in 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: mgrmap e40: smithi174.whsozo(active, since 18s), standbys: smithi184.prgadw 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: overall HEALTH_OK 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:40.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:39 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:27:40.832 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:40 smithi174 ceph-mon[66222]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 480 op/s 2024-02-07T12:27:41.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 173024 11.59 MB/sec execute 208 sec latency 45.711 ms 2024-02-07T12:27:41.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:40 smithi184 ceph-mon[53060]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 480 op/s 2024-02-07T12:27:42.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 173306 11.58 MB/sec execute 209 sec latency 48.060 ms 2024-02-07T12:27:43.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 173670 11.60 MB/sec execute 210 sec latency 42.745 ms 2024-02-07T12:27:43.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:43 smithi174 ceph-mon[66222]: pgmap v14: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 673 op/s 2024-02-07T12:27:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:43 smithi184 ceph-mon[53060]: pgmap v14: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 673 op/s 2024-02-07T12:27:44.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 174025 11.60 MB/sec execute 211 sec latency 40.164 ms 2024-02-07T12:27:44.461 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:44 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:44.461 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:44 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:44 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:44 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:45.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 174342 11.60 MB/sec execute 212 sec latency 52.720 ms 2024-02-07T12:27:45.487 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:45 smithi174 ceph-mon[66222]: pgmap v15: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 391 op/s 2024-02-07T12:27:45.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:45 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:45.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:45 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:45.505 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:45 smithi184 ceph-mon[53060]: pgmap v15: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 391 op/s 2024-02-07T12:27:45.505 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:45 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:45.506 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:45 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:46.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 174608 11.57 MB/sec execute 213 sec latency 51.303 ms 2024-02-07T12:27:47.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 174796 11.52 MB/sec execute 214 sec latency 50.915 ms 2024-02-07T12:27:47.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:47 smithi174 ceph-mon[66222]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 391 op/s 2024-02-07T12:27:47.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:47 smithi184 ceph-mon[53060]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 391 op/s 2024-02-07T12:27:48.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 175067 11.50 MB/sec execute 215 sec latency 56.290 ms 2024-02-07T12:27:49.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 175284 11.47 MB/sec execute 216 sec latency 42.206 ms 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.0 MiB/s wr, 576 op/s 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:27:49.619 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.0 MiB/s wr, 576 op/s 2024-02-07T12:27:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:27:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:27:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:27:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:50.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 175560 11.46 MB/sec execute 217 sec latency 65.241 ms 2024-02-07T12:27:50.625 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:50 smithi174 ceph-mon[66222]: Reconfiguring mon.smithi174 (monmap changed)... 2024-02-07T12:27:50.625 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:50 smithi174 ceph-mon[66222]: Reconfiguring daemon mon.smithi174 on smithi174 2024-02-07T12:27:50.626 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:27:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:50 smithi184 ceph-mon[53060]: Reconfiguring mon.smithi174 (monmap changed)... 2024-02-07T12:27:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:50 smithi184 ceph-mon[53060]: Reconfiguring daemon mon.smithi174 on smithi174 2024-02-07T12:27:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:27:51.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 176020 11.46 MB/sec execute 218 sec latency 37.254 ms 2024-02-07T12:27:51.641 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:51 smithi174 ceph-mon[66222]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-02-07T12:27:51.642 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:51.642 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:51.642 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.whsozo", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:27:51.642 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:27:51.642 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:51 smithi184 ceph-mon[53060]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-02-07T12:27:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.whsozo", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:27:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:27:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:52.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 176399 11.43 MB/sec execute 219 sec latency 43.009 ms 2024-02-07T12:27:52.419 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:52 smithi174 ceph-mon[66222]: Reconfiguring mgr.smithi174.whsozo (monmap changed)... 2024-02-07T12:27:52.419 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:52 smithi174 ceph-mon[66222]: Reconfiguring daemon mgr.smithi174.whsozo on smithi174 2024-02-07T12:27:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:52 smithi184 ceph-mon[53060]: Reconfiguring mgr.smithi174.whsozo (monmap changed)... 2024-02-07T12:27:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:52 smithi184 ceph-mon[53060]: Reconfiguring daemon mgr.smithi174.whsozo on smithi174 2024-02-07T12:27:53.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 176739 11.41 MB/sec execute 220 sec latency 51.566 ms 2024-02-07T12:27:53.422 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:53 smithi174 ceph-mon[66222]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 564 op/s 2024-02-07T12:27:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:53 smithi184 ceph-mon[53060]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 564 op/s 2024-02-07T12:27:54.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 177055 11.42 MB/sec execute 221 sec latency 73.487 ms 2024-02-07T12:27:54.045 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:27:54.936 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:54.937 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:54.937 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:54 smithi174 ceph-mon[66222]: Reconfiguring crash.smithi174 (monmap changed)... 2024-02-07T12:27:54.937 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:27:54.937 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:54.937 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:54 smithi174 ceph-mon[66222]: Reconfiguring daemon crash.smithi174 on smithi174 2024-02-07T12:27:54.937 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:54 smithi174 ceph-mon[66222]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-02-07T12:27:54.937 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:54 smithi174 ceph-mon[66222]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:55.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 177373 11.43 MB/sec execute 222 sec latency 58.713 ms 2024-02-07T12:27:55.063 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:27:55.063 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (3m) 25s ago 12m 23.8M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:27:55.063 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (12m) 25s ago 12m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:27:55.063 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (11m) 11s ago 11m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:27:55.063 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (2m) 25s ago 11m 44.8M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:27:55.063 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (6m) 25s ago 6m 38.1M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (6m) 11s ago 6m 13.0M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (119s) 25s ago 13m 477M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (66s) 11s ago 11m 447M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (47s) 25s ago 13m 32.0M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (17s) 11s ago 11m 25.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (5m) 25s ago 5m 260M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (5m) 11s ago 5m 82.4M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (2m) 25s ago 11m 10.4M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (2m) 11s ago 11m 18.4M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (10m) 25s ago 10m 326M 1936M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (10m) 25s ago 10m 484M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (9m) 25s ago 9m 368M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (9m) 25s ago 9m 515M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:27:55.064 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (8m) 11s ago 8m 552M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:27:55.065 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (8m) 11s ago 8m 379M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:27:55.065 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (8m) 11s ago 8m 554M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:27:55.065 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (7m) 11s ago 7m 462M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:27:55.065 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (73s) 25s ago 10m 55.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:27:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:54 smithi184 ceph-mon[53060]: Reconfiguring crash.smithi174 (monmap changed)... 2024-02-07T12:27:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:27:55.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:55.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:54 smithi184 ceph-mon[53060]: Reconfiguring daemon crash.smithi174 on smithi174 2024-02-07T12:27:55.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:54 smithi184 ceph-mon[53060]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-02-07T12:27:55.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:54 smithi184 ceph-mon[53060]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:55.675 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:27:55.676 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-02-07T12:27:55.677 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 4 2024-02-07T12:27:55.677 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:27:55.677 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:27:55.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:55 smithi174 ceph-mon[66222]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:55.960 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:55 smithi174 ceph-mon[66222]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:55.960 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:55 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1715260836' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:56.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 177700 11.44 MB/sec execute 223 sec latency 43.786 ms 2024-02-07T12:27:56.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:55 smithi184 ceph-mon[53060]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:56.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:55 smithi184 ceph-mon[53060]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:56.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:55 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1715260836' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "4/23 daemons upgraded", 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading mon daemons", 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:27:56.200 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:27:56.792 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:27:56.986 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:56 smithi174 ceph-mon[66222]: pgmap v21: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-02-07T12:27:56.987 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:56 smithi174 ceph-mon[66222]: from='client.34166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:56.987 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:56 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:56.987 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:56 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3101667047' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:27:56.987 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:56 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:56.987 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:56 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:27:56.987 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:56 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:57.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 177962 11.42 MB/sec execute 224 sec latency 58.392 ms 2024-02-07T12:27:57.117 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:56 smithi184 ceph-mon[53060]: pgmap v21: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-02-07T12:27:57.118 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:56 smithi184 ceph-mon[53060]: from='client.34166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:27:57.118 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:56 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:57.118 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:56 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3101667047' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:27:57.118 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:56 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:27:57.118 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:56 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:27:57.118 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:56 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:27:58.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:57 smithi174 ceph-mon[66222]: Reconfiguring osd.0 (monmap changed)... 2024-02-07T12:27:58.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:57 smithi174 ceph-mon[66222]: Reconfiguring daemon osd.0 on smithi174 2024-02-07T12:27:58.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 178206 11.38 MB/sec execute 225 sec latency 42.929 ms 2024-02-07T12:27:58.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:57 smithi184 ceph-mon[53060]: Reconfiguring osd.0 (monmap changed)... 2024-02-07T12:27:58.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:57 smithi184 ceph-mon[53060]: Reconfiguring daemon osd.0 on smithi174 2024-02-07T12:27:59.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 178496 11.35 MB/sec execute 226 sec latency 42.155 ms 2024-02-07T12:27:59.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:27:58 smithi174 ceph-mon[66222]: pgmap v22: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.0 MiB/s wr, 566 op/s 2024-02-07T12:27:59.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:27:58 smithi184 ceph-mon[53060]: pgmap v22: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.0 MiB/s wr, 566 op/s 2024-02-07T12:28:00.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 178711 11.32 MB/sec execute 227 sec latency 61.214 ms 2024-02-07T12:28:00.578 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:00.579 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:00.579 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:28:00.579 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:28:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:01.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 179150 11.34 MB/sec execute 228 sec latency 43.488 ms 2024-02-07T12:28:01.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:01 smithi174 ceph-mon[66222]: Reconfiguring osd.1 (monmap changed)... 2024-02-07T12:28:01.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:01 smithi174 ceph-mon[66222]: Reconfiguring daemon osd.1 on smithi174 2024-02-07T12:28:01.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:01 smithi174 ceph-mon[66222]: pgmap v23: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 381 op/s 2024-02-07T12:28:01.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:01 smithi184 ceph-mon[53060]: Reconfiguring osd.1 (monmap changed)... 2024-02-07T12:28:01.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:01 smithi184 ceph-mon[53060]: Reconfiguring daemon osd.1 on smithi174 2024-02-07T12:28:01.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:01 smithi184 ceph-mon[53060]: pgmap v23: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 381 op/s 2024-02-07T12:28:02.024 INFO:teuthology.orchestra.run.smithi174.stdout: 5 179649 11.33 MB/sec execute 229 sec latency 76.024 ms 2024-02-07T12:28:03.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 180126 11.31 MB/sec execute 230 sec latency 35.320 ms 2024-02-07T12:28:03.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:03 smithi174 ceph-mon[66222]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.7 MiB/s wr, 580 op/s 2024-02-07T12:28:03.534 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:03 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:03.535 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:03 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:03.535 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:03 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:28:03.535 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:03 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:03.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:03 smithi184 ceph-mon[53060]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.7 MiB/s wr, 580 op/s 2024-02-07T12:28:03.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:03 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:03.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:03 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:03.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:03 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:28:03.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:03 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:04.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 180690 11.36 MB/sec execute 231 sec latency 36.564 ms 2024-02-07T12:28:04.551 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:04 smithi174 ceph-mon[66222]: Reconfiguring osd.2 (monmap changed)... 2024-02-07T12:28:04.552 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:04 smithi174 ceph-mon[66222]: Reconfiguring daemon osd.2 on smithi174 2024-02-07T12:28:04.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:04 smithi184 ceph-mon[53060]: Reconfiguring osd.2 (monmap changed)... 2024-02-07T12:28:04.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:04 smithi184 ceph-mon[53060]: Reconfiguring daemon osd.2 on smithi174 2024-02-07T12:28:05.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 181229 11.39 MB/sec execute 232 sec latency 32.096 ms 2024-02-07T12:28:05.569 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:05 smithi174 ceph-mon[66222]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-02-07T12:28:05.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:28:05.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:05.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:05.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:28:05.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:05.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:05 smithi184 ceph-mon[53060]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-02-07T12:28:05.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:28:05.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:05.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:05.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:28:05.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:06.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 181622 11.40 MB/sec execute 233 sec latency 34.491 ms 2024-02-07T12:28:06.577 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:06 smithi174 ceph-mon[66222]: Reconfiguring osd.3 (monmap changed)... 2024-02-07T12:28:06.577 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:06 smithi174 ceph-mon[66222]: Reconfiguring daemon osd.3 on smithi174 2024-02-07T12:28:06.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:06 smithi184 ceph-mon[53060]: Reconfiguring osd.3 (monmap changed)... 2024-02-07T12:28:06.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:06 smithi184 ceph-mon[53060]: Reconfiguring daemon osd.3 on smithi174 2024-02-07T12:28:07.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 181962 11.37 MB/sec execute 234 sec latency 45.193 ms 2024-02-07T12:28:07.589 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:07 smithi174 ceph-mon[66222]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-02-07T12:28:07.834 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:07 smithi184 ceph-mon[53060]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-02-07T12:28:08.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 182348 11.35 MB/sec execute 235 sec latency 41.546 ms 2024-02-07T12:28:09.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 182731 11.36 MB/sec execute 236 sec latency 64.526 ms 2024-02-07T12:28:09.100 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:09 smithi174 ceph-mon[66222]: pgmap v27: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 675 op/s 2024-02-07T12:28:09.100 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:09.101 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:09.101 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:09 smithi174 ceph-mon[66222]: Reconfiguring mds.foofs.smithi174.vkrzkp (monmap changed)... 2024-02-07T12:28:09.101 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.vkrzkp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:28:09.101 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:09.101 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:09 smithi174 ceph-mon[66222]: Reconfiguring daemon mds.foofs.smithi174.vkrzkp on smithi174 2024-02-07T12:28:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:09 smithi184 ceph-mon[53060]: pgmap v27: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 675 op/s 2024-02-07T12:28:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:09 smithi184 ceph-mon[53060]: Reconfiguring mds.foofs.smithi174.vkrzkp (monmap changed)... 2024-02-07T12:28:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.vkrzkp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:28:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:09.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:09 smithi184 ceph-mon[53060]: Reconfiguring daemon mds.foofs.smithi174.vkrzkp on smithi174 2024-02-07T12:28:10.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 183231 11.35 MB/sec execute 237 sec latency 50.124 ms 2024-02-07T12:28:11.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 183684 11.33 MB/sec execute 238 sec latency 49.196 ms 2024-02-07T12:28:11.122 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:11 smithi174 ceph-mon[66222]: pgmap v28: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 480 op/s 2024-02-07T12:28:11.123 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:11 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:11.123 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:11 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:11.123 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:11 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:28:11.123 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:11 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:11.289 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:11 smithi184 ceph-mon[53060]: pgmap v28: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 480 op/s 2024-02-07T12:28:11.289 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:11 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:11.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:11 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:11.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:11 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:28:11.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:11 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:12.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 184095 11.35 MB/sec execute 239 sec latency 36.892 ms 2024-02-07T12:28:12.134 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:12 smithi174 ceph-mon[66222]: Reconfiguring crash.smithi184 (monmap changed)... 2024-02-07T12:28:12.134 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:12 smithi174 ceph-mon[66222]: Reconfiguring daemon crash.smithi184 on smithi184 2024-02-07T12:28:12.321 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:12 smithi184 ceph-mon[53060]: Reconfiguring crash.smithi184 (monmap changed)... 2024-02-07T12:28:12.321 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:12 smithi184 ceph-mon[53060]: Reconfiguring daemon crash.smithi184 on smithi184 2024-02-07T12:28:13.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 184596 11.39 MB/sec execute 240 sec latency 54.183 ms 2024-02-07T12:28:13.395 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:13 smithi174 ceph-mon[66222]: pgmap v29: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 741 op/s 2024-02-07T12:28:13.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:13.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:13.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:13 smithi174 ceph-mon[66222]: Reconfiguring mgr.smithi184.prgadw (monmap changed)... 2024-02-07T12:28:13.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:28:13.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:28:13.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:13.396 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:13 smithi174 ceph-mon[66222]: Reconfiguring daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:28:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:13 smithi184 ceph-mon[53060]: pgmap v29: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 741 op/s 2024-02-07T12:28:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:13 smithi184 ceph-mon[53060]: Reconfiguring mgr.smithi184.prgadw (monmap changed)... 2024-02-07T12:28:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi184.prgadw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-02-07T12:28:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mgr services"}]: dispatch 2024-02-07T12:28:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:13.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:13 smithi184 ceph-mon[53060]: Reconfiguring daemon mgr.smithi184.prgadw on smithi184 2024-02-07T12:28:14.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 185130 11.41 MB/sec execute 241 sec latency 37.858 ms 2024-02-07T12:28:14.902 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:14 smithi174 ceph-mon[66222]: pgmap v30: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 541 op/s 2024-02-07T12:28:15.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 185631 11.40 MB/sec execute 242 sec latency 100.057 ms 2024-02-07T12:28:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:14 smithi184 ceph-mon[53060]: pgmap v30: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 541 op/s 2024-02-07T12:28:15.901 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:15.901 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:15.901 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:15 smithi184 ceph-mon[53060]: Reconfiguring mon.smithi184 (monmap changed)... 2024-02-07T12:28:15.901 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:28:15.901 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:28:15.901 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:15.901 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:15 smithi184 ceph-mon[53060]: Reconfiguring daemon mon.smithi184 on smithi184 2024-02-07T12:28:15.911 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:15.911 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:15.911 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:15 smithi174 ceph-mon[66222]: Reconfiguring mon.smithi184 (monmap changed)... 2024-02-07T12:28:15.911 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-02-07T12:28:15.911 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-02-07T12:28:15.911 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:15.911 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:15 smithi174 ceph-mon[66222]: Reconfiguring daemon mon.smithi184 on smithi184 2024-02-07T12:28:16.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 186430 11.46 MB/sec execute 243 sec latency 26.921 ms 2024-02-07T12:28:16.921 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:16 smithi174 ceph-mon[66222]: pgmap v31: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 541 op/s 2024-02-07T12:28:16.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:16 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:16.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:16 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:16.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:16 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:28:16.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:16 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:16.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:16 smithi184 ceph-mon[53060]: pgmap v31: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 541 op/s 2024-02-07T12:28:16.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:16 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:16.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:16 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:16.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:16 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:28:16.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:16 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:17.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 187080 11.43 MB/sec execute 244 sec latency 51.072 ms 2024-02-07T12:28:17.938 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:17 smithi174 ceph-mon[66222]: Reconfiguring osd.4 (monmap changed)... 2024-02-07T12:28:17.938 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:17 smithi174 ceph-mon[66222]: Reconfiguring daemon osd.4 on smithi184 2024-02-07T12:28:17.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:17 smithi184 ceph-mon[53060]: Reconfiguring osd.4 (monmap changed)... 2024-02-07T12:28:17.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:17 smithi184 ceph-mon[53060]: Reconfiguring daemon osd.4 on smithi184 2024-02-07T12:28:18.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 187313 11.40 MB/sec execute 245 sec latency 39.572 ms 2024-02-07T12:28:18.984 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:18 smithi174 ceph-mon[66222]: pgmap v32: 97 pgs: 97 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 876 op/s 2024-02-07T12:28:18.984 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:18 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:18.984 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:18 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:18.984 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:18 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:28:18.984 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:18 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:19.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 187594 11.41 MB/sec execute 246 sec latency 42.071 ms 2024-02-07T12:28:19.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:18 smithi184 ceph-mon[53060]: pgmap v32: 97 pgs: 97 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 876 op/s 2024-02-07T12:28:19.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:18 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:19.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:18 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:19.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:18 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:28:19.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:18 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:20.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 187958 11.42 MB/sec execute 247 sec latency 60.001 ms 2024-02-07T12:28:20.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:19 smithi174 ceph-mon[66222]: Reconfiguring osd.5 (monmap changed)... 2024-02-07T12:28:20.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:19 smithi174 ceph-mon[66222]: Reconfiguring daemon osd.5 on smithi184 2024-02-07T12:28:20.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:19 smithi184 ceph-mon[53060]: Reconfiguring osd.5 (monmap changed)... 2024-02-07T12:28:20.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:19 smithi184 ceph-mon[53060]: Reconfiguring daemon osd.5 on smithi184 2024-02-07T12:28:20.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:20 smithi184 ceph-mon[53060]: pgmap v33: 97 pgs: 97 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 595 op/s 2024-02-07T12:28:20.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:28:20.990 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:20 smithi174 ceph-mon[66222]: pgmap v33: 97 pgs: 97 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 595 op/s 2024-02-07T12:28:20.990 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:28:21.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 188509 11.46 MB/sec execute 248 sec latency 59.109 ms 2024-02-07T12:28:22.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 189055 11.46 MB/sec execute 249 sec latency 28.555 ms 2024-02-07T12:28:22.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:22 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:22.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:22 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:22.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:22 smithi184 ceph-mon[53060]: Reconfiguring osd.6 (monmap changed)... 2024-02-07T12:28:22.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:22 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:28:22.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:22 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:22.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:22 smithi184 ceph-mon[53060]: Reconfiguring daemon osd.6 on smithi184 2024-02-07T12:28:22.332 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:22 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:22.333 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:22 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:22.333 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:22 smithi174 ceph-mon[66222]: Reconfiguring osd.6 (monmap changed)... 2024-02-07T12:28:22.333 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:22 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:28:22.333 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:22 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:22.333 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:22 smithi174 ceph-mon[66222]: Reconfiguring daemon osd.6 on smithi184 2024-02-07T12:28:23.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 189394 11.43 MB/sec execute 250 sec latency 48.334 ms 2024-02-07T12:28:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:23 smithi184 ceph-mon[53060]: pgmap v34: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 834 op/s 2024-02-07T12:28:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:23 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:23 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:23 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:28:23.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:23 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:23.350 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:23 smithi174 ceph-mon[66222]: pgmap v34: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 834 op/s 2024-02-07T12:28:23.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:23 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:23.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:23 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:23.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:23 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:28:23.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:23 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:24.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 189637 11.42 MB/sec execute 251 sec latency 53.477 ms 2024-02-07T12:28:24.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:24 smithi184 ceph-mon[53060]: Reconfiguring osd.7 (monmap changed)... 2024-02-07T12:28:24.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:24 smithi184 ceph-mon[53060]: Reconfiguring daemon osd.7 on smithi184 2024-02-07T12:28:24.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:24 smithi174 ceph-mon[66222]: Reconfiguring osd.7 (monmap changed)... 2024-02-07T12:28:24.361 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:24 smithi174 ceph-mon[66222]: Reconfiguring daemon osd.7 on smithi184 2024-02-07T12:28:25.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 189975 11.43 MB/sec execute 252 sec latency 50.491 ms 2024-02-07T12:28:25.127 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:25 smithi174 ceph-mon[66222]: pgmap v35: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 573 op/s 2024-02-07T12:28:25.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:25.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:25.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi184.alpixp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:28:25.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:25.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:25 smithi184 ceph-mon[53060]: pgmap v35: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 573 op/s 2024-02-07T12:28:25.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:25.386 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:25.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi184.alpixp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:28:25.387 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:26.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 190575 11.40 MB/sec execute 253 sec latency 52.011 ms 2024-02-07T12:28:26.121 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:26 smithi174 ceph-mon[66222]: Reconfiguring mds.foofs.smithi184.alpixp (monmap changed)... 2024-02-07T12:28:26.121 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:26 smithi174 ceph-mon[66222]: Reconfiguring daemon mds.foofs.smithi184.alpixp on smithi184 2024-02-07T12:28:26.389 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:26 smithi184 ceph-mon[53060]: Reconfiguring mds.foofs.smithi184.alpixp (monmap changed)... 2024-02-07T12:28:26.390 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:26 smithi184 ceph-mon[53060]: Reconfiguring daemon mds.foofs.smithi184.alpixp on smithi184 2024-02-07T12:28:27.025 INFO:teuthology.orchestra.run.smithi174.stdout: 5 191147 11.42 MB/sec execute 254 sec latency 31.101 ms 2024-02-07T12:28:27.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 573 op/s 2024-02-07T12:28:27.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]: dispatch 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]': finished 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi184"}]: dispatch 2024-02-07T12:28:27.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi184"}]': finished 2024-02-07T12:28:27.355 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:28:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 573 op/s 2024-02-07T12:28:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:28:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:28:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:28:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]: dispatch 2024-02-07T12:28:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]': finished 2024-02-07T12:28:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi184"}]: dispatch 2024-02-07T12:28:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi184"}]': finished 2024-02-07T12:28:28.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 191815 11.47 MB/sec execute 255 sec latency 50.974 ms 2024-02-07T12:28:28.142 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:28 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all mon 2024-02-07T12:28:28.142 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:28 smithi174 ceph-mon[66222]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:28 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all mon 2024-02-07T12:28:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:28 smithi184 ceph-mon[53060]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:28.486 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:28:28.486 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (3m) 58s ago 12m 23.8M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:28:28.486 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (12m) 58s ago 12m 6409k - 17.2.0 e1d6a67b021e c7dff5ac9685 2024-02-07T12:28:28.486 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (12m) 45s ago 12m 6405k - 17.2.0 e1d6a67b021e 704a4691f488 2024-02-07T12:28:28.486 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (3m) 58s ago 12m 44.8M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:28:28.486 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (6m) 58s ago 6m 38.1M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:28:28.486 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (6m) 45s ago 6m 13.0M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (2m) 58s ago 14m 477M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (100s) 45s ago 12m 447M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (80s) 58s ago 14m 32.0M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (50s) 45s ago 11m 25.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (6m) 58s ago 6m 260M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (6m) 45s ago 6m 82.4M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (3m) 58s ago 12m 10.4M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (2m) 45s ago 11m 18.4M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (11m) 58s ago 11m 326M 1936M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (10m) 58s ago 10m 484M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (10m) 58s ago 10m 368M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (9m) 58s ago 9m 515M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (9m) 45s ago 9m 552M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (9m) 45s ago 9m 379M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:28:28.487 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (8m) 45s ago 8m 554M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:28:28.488 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (8m) 45s ago 8m 462M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:28:28.488 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (107s) 58s ago 11m 55.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:28:29.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 192278 11.49 MB/sec execute 256 sec latency 29.414 ms 2024-02-07T12:28:29.091 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:28:29.091 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:28:29.091 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:28:29.091 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:28:29.091 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:28:29.091 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 4 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:28:29.092 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:28:29.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:29 smithi174 ceph-mon[66222]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:29.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:29 smithi174 ceph-mon[66222]: pgmap v37: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 835 op/s 2024-02-07T12:28:29.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:29 smithi174 ceph-mon[66222]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:29.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:29 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3657501765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:28:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:29 smithi184 ceph-mon[53060]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:29.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:29 smithi184 ceph-mon[53060]: pgmap v37: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 835 op/s 2024-02-07T12:28:29.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:29 smithi184 ceph-mon[53060]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:29.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:29 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3657501765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "4/23 daemons upgraded", 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading crash daemons", 2024-02-07T12:28:29.597 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:28:29.598 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:28:30.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 192630 11.48 MB/sec execute 257 sec latency 52.748 ms 2024-02-07T12:28:30.332 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:28:30.689 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:30 smithi174 ceph-mon[66222]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:30.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:30 smithi174 ceph-mon[66222]: Upgrade: Updating crash.smithi174 (1/2) 2024-02-07T12:28:30.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:30.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:28:30.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:30.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:30 smithi174 ceph-mon[66222]: Deploying daemon crash.smithi174 on smithi174 2024-02-07T12:28:30.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:30 smithi174 ceph-mon[66222]: pgmap v38: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.7 MiB/s wr, 500 op/s 2024-02-07T12:28:30.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:30 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1819788108' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:28:30.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:30 smithi184 ceph-mon[53060]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:28:30.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:30 smithi184 ceph-mon[53060]: Upgrade: Updating crash.smithi174 (1/2) 2024-02-07T12:28:30.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:30.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:28:30.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:30.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:30 smithi184 ceph-mon[53060]: Deploying daemon crash.smithi174 on smithi174 2024-02-07T12:28:30.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:30 smithi184 ceph-mon[53060]: pgmap v38: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.7 MiB/s wr, 500 op/s 2024-02-07T12:28:30.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:30 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1819788108' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:28:31.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 193014 11.46 MB/sec execute 258 sec latency 48.818 ms 2024-02-07T12:28:32.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 193379 11.45 MB/sec execute 259 sec latency 38.310 ms 2024-02-07T12:28:33.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 193722 11.45 MB/sec execute 260 sec latency 75.788 ms 2024-02-07T12:28:33.245 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:33 smithi174 ceph-mon[66222]: pgmap v39: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 799 op/s 2024-02-07T12:28:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:33 smithi184 ceph-mon[53060]: pgmap v39: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 799 op/s 2024-02-07T12:28:34.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 194055 11.41 MB/sec execute 261 sec latency 66.178 ms 2024-02-07T12:28:34.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:34 smithi174 ceph-mon[66222]: pgmap v40: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 560 op/s 2024-02-07T12:28:35.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 194338 11.39 MB/sec execute 262 sec latency 49.321 ms 2024-02-07T12:28:35.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:34 smithi184 ceph-mon[53060]: pgmap v40: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 560 op/s 2024-02-07T12:28:35.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:28:36.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 194517 11.37 MB/sec execute 263 sec latency 49.493 ms 2024-02-07T12:28:36.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:28:36.857 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:36.857 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:36.858 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:36 smithi174 ceph-mon[66222]: pgmap v41: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 560 op/s 2024-02-07T12:28:37.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 194837 11.37 MB/sec execute 264 sec latency 57.841 ms 2024-02-07T12:28:37.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:37.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:37.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:36 smithi184 ceph-mon[53060]: pgmap v41: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 560 op/s 2024-02-07T12:28:38.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 195202 11.39 MB/sec execute 265 sec latency 66.239 ms 2024-02-07T12:28:38.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:38 smithi174 ceph-mon[66222]: Upgrade: Updating crash.smithi184 (2/2) 2024-02-07T12:28:38.691 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:38 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:38.691 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:38 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:28:38.691 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:38 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:38.691 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:38 smithi174 ceph-mon[66222]: Deploying daemon crash.smithi184 on smithi184 2024-02-07T12:28:38.691 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:38 smithi174 ceph-mon[66222]: pgmap v42: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 757 op/s 2024-02-07T12:28:38.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:38 smithi184 ceph-mon[53060]: Upgrade: Updating crash.smithi184 (2/2) 2024-02-07T12:28:38.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:38 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:38.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:38 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi184", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-02-07T12:28:38.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:38 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:28:38.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:38 smithi184 ceph-mon[53060]: Deploying daemon crash.smithi184 on smithi184 2024-02-07T12:28:38.879 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:38 smithi184 ceph-mon[53060]: pgmap v42: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 757 op/s 2024-02-07T12:28:39.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 195561 11.39 MB/sec execute 266 sec latency 59.060 ms 2024-02-07T12:28:40.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 195772 11.37 MB/sec execute 267 sec latency 76.686 ms 2024-02-07T12:28:41.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 196063 11.36 MB/sec execute 268 sec latency 40.299 ms 2024-02-07T12:28:41.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:41 smithi184 ceph-mon[53060]: pgmap v43: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-02-07T12:28:41.266 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:41 smithi174 ceph-mon[66222]: pgmap v43: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-02-07T12:28:42.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 196471 11.34 MB/sec execute 269 sec latency 40.791 ms 2024-02-07T12:28:43.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 196769 11.34 MB/sec execute 270 sec latency 44.709 ms 2024-02-07T12:28:43.031 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:42 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:43.031 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:42 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:43.031 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:42 smithi174 ceph-mon[66222]: pgmap v44: 97 pgs: 97 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.3 MiB/s wr, 672 op/s 2024-02-07T12:28:43.032 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:42 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:28:43.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:42 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:43.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:42 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:43.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:42 smithi184 ceph-mon[53060]: pgmap v44: 97 pgs: 97 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.3 MiB/s wr, 672 op/s 2024-02-07T12:28:43.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:42 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:28:44.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 197081 11.32 MB/sec execute 271 sec latency 81.290 ms 2024-02-07T12:28:44.827 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:44 smithi174 ceph-mon[66222]: pgmap v45: 97 pgs: 97 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-02-07T12:28:44.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:44 smithi184 ceph-mon[53060]: pgmap v45: 97 pgs: 97 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-02-07T12:28:45.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 197416 11.30 MB/sec execute 272 sec latency 84.284 ms 2024-02-07T12:28:46.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 197794 11.27 MB/sec execute 273 sec latency 59.662 ms 2024-02-07T12:28:47.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 198044 11.26 MB/sec execute 274 sec latency 52.593 ms 2024-02-07T12:28:47.198 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:47 smithi174 ceph-mon[66222]: pgmap v46: 97 pgs: 97 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-02-07T12:28:47.198 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:47 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:47.198 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:47 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:47 smithi184 ceph-mon[53060]: pgmap v46: 97 pgs: 97 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-02-07T12:28:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:47 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:47 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:48.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 198480 11.29 MB/sec execute 275 sec latency 40.190 ms 2024-02-07T12:28:48.615 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:48 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:48.616 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:48 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:48.616 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:48 smithi184 ceph-mon[53060]: pgmap v47: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 590 op/s 2024-02-07T12:28:48.717 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:48 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:48.717 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:48 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:48.717 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:48 smithi174 ceph-mon[66222]: pgmap v47: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 590 op/s 2024-02-07T12:28:49.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 198981 11.31 MB/sec execute 276 sec latency 30.149 ms 2024-02-07T12:28:50.026 INFO:teuthology.orchestra.run.smithi174.stdout: 5 199321 11.31 MB/sec execute 277 sec latency 49.046 ms 2024-02-07T12:28:51.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 199664 11.30 MB/sec execute 278 sec latency 45.337 ms 2024-02-07T12:28:51.254 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:51 smithi174 ceph-mon[66222]: pgmap v48: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 393 op/s 2024-02-07T12:28:51.254 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:28:51.416 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:51 smithi184 ceph-mon[53060]: pgmap v48: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 393 op/s 2024-02-07T12:28:51.416 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:28:52.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 200082 11.28 MB/sec execute 279 sec latency 42.543 ms 2024-02-07T12:28:52.282 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:52 smithi174 ceph-mon[66222]: mgrmap e41: smithi174.whsozo(active, since 91s), standbys: smithi184.prgadw 2024-02-07T12:28:52.283 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:52.283 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:52.283 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:52.283 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:52.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:52 smithi184 ceph-mon[53060]: mgrmap e41: smithi174.whsozo(active, since 91s), standbys: smithi184.prgadw 2024-02-07T12:28:52.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:52.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:52.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:52.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:53.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 200384 11.28 MB/sec execute 280 sec latency 56.039 ms 2024-02-07T12:28:53.554 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:53 smithi174 ceph-mon[66222]: pgmap v49: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-02-07T12:28:53.554 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:53.555 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:53 smithi184 ceph-mon[53060]: pgmap v49: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-02-07T12:28:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:28:54.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 200798 11.27 MB/sec execute 281 sec latency 38.307 ms 2024-02-07T12:28:54.818 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:54 smithi174 ceph-mon[66222]: pgmap v50: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-02-07T12:28:55.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 201292 11.25 MB/sec execute 282 sec latency 49.506 ms 2024-02-07T12:28:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:54 smithi184 ceph-mon[53060]: pgmap v50: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-02-07T12:28:56.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 201617 11.25 MB/sec execute 283 sec latency 33.336 ms 2024-02-07T12:28:57.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 202077 11.28 MB/sec execute 284 sec latency 34.779 ms 2024-02-07T12:28:57.089 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:57 smithi174 ceph-mon[66222]: pgmap v51: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-02-07T12:28:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:57 smithi184 ceph-mon[53060]: pgmap v51: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-02-07T12:28:58.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 202480 11.28 MB/sec execute 285 sec latency 41.955 ms 2024-02-07T12:28:59.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 202833 11.28 MB/sec execute 286 sec latency 48.318 ms 2024-02-07T12:28:59.079 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:28:59 smithi174 ceph-mon[66222]: pgmap v52: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 699 op/s 2024-02-07T12:28:59.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:28:59 smithi184 ceph-mon[53060]: pgmap v52: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 699 op/s 2024-02-07T12:29:00.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 203027 11.26 MB/sec execute 287 sec latency 67.103 ms 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:00.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all crash 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]: dispatch 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]': finished 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi184"}]: dispatch 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi184"}]': finished 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:29:00.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:00 smithi174 ceph-mon[66222]: Upgrade: osd.0 is safe to restart 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all crash 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:00.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]: dispatch 2024-02-07T12:29:00.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]': finished 2024-02-07T12:29:00.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi184"}]: dispatch 2024-02-07T12:29:00.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi184"}]': finished 2024-02-07T12:29:00.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:29:00.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-02-07T12:29:00.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:00 smithi184 ceph-mon[53060]: Upgrade: osd.0 is safe to restart 2024-02-07T12:29:00.852 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:29:01.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 203244 11.24 MB/sec execute 288 sec latency 44.298 ms 2024-02-07T12:29:01.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:01 smithi174 ceph-mon[66222]: pgmap v53: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-02-07T12:29:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:01 smithi184 ceph-mon[53060]: pgmap v53: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-02-07T12:29:01.864 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:29:01.864 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (4m) 10s ago 13m 22.5M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:29:01.864 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (26s) 10s ago 13m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (20s) 15s ago 12m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (3m) 10s ago 13m 53.0M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (7m) 10s ago 7m 38.7M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (7m) 15s ago 7m 13.3M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (3m) 10s ago 14m 499M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (2m) 15s ago 12m 448M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (113s) 10s ago 14m 42.5M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (84s) 15s ago 12m 35.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (7m) 10s ago 7m 259M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (6m) 15s ago 6m 82.5M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (3m) 10s ago 12m 16.9M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (3m) 15s ago 12m 18.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (11m) 10s ago 11m 411M 1936M 17.2.0 e1d6a67b021e 948aa062fcd5 2024-02-07T12:29:01.865 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (11m) 10s ago 11m 583M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:29:01.866 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (10m) 10s ago 10m 497M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:29:01.866 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (10m) 10s ago 10m 607M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:29:01.866 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (10m) 15s ago 10m 575M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:29:01.866 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (9m) 15s ago 9m 442M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:29:01.866 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (9m) 15s ago 9m 343M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:29:01.866 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (8m) 15s ago 8m 546M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:29:01.866 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (2m) 10s ago 12m 59.8M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:29:02.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 203495 11.21 MB/sec execute 289 sec latency 98.581 ms 2024-02-07T12:29:02.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:02 smithi174 ceph-mon[66222]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:02.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:02 smithi174 ceph-mon[66222]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:02.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:02.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:29:02.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:02.403 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:29:02.404 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-02-07T12:29:02.405 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 4 2024-02-07T12:29:02.405 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:29:02.405 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:29:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:02 smithi184 ceph-mon[53060]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:02 smithi184 ceph-mon[53060]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-02-07T12:29:02.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:02.944 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:29:02.944 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:29:02.944 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:29:02.944 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:29:02.944 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:29:02.945 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:29:02.945 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:29:02.945 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:29:02.945 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:29:02.945 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "6/23 daemons upgraded", 2024-02-07T12:29:02.945 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:29:02.945 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:29:02.945 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:29:03.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 203706 11.19 MB/sec execute 290 sec latency 74.696 ms 2024-02-07T12:29:03.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:03 smithi174 ceph-mon[66222]: Upgrade: Updating osd.0 2024-02-07T12:29:03.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:03 smithi174 ceph-mon[66222]: Deploying daemon osd.0 on smithi174 2024-02-07T12:29:03.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:03 smithi174 ceph-mon[66222]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:03.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:03 smithi174 ceph-mon[66222]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 669 op/s 2024-02-07T12:29:03.404 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:03 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/2113197253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:03.405 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:29:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:03 smithi184 ceph-mon[53060]: Upgrade: Updating osd.0 2024-02-07T12:29:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:03 smithi184 ceph-mon[53060]: Deploying daemon osd.0 on smithi174 2024-02-07T12:29:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:03 smithi184 ceph-mon[53060]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:03 smithi184 ceph-mon[53060]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 669 op/s 2024-02-07T12:29:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:03 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/2113197253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:04.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 203935 11.18 MB/sec execute 291 sec latency 64.243 ms 2024-02-07T12:29:04.429 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:04 smithi174 ceph-mon[66222]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:04.430 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:04 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/4166994438' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:29:04.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:04 smithi184 ceph-mon[53060]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:04.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:04 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/4166994438' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:29:05.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 204260 11.17 MB/sec execute 292 sec latency 64.686 ms 2024-02-07T12:29:05.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:05 smithi174 ceph-mon[66222]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-02-07T12:29:05.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:29:05.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:05 smithi184 ceph-mon[53060]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-02-07T12:29:05.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:29:06.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 204556 11.14 MB/sec execute 293 sec latency 68.117 ms 2024-02-07T12:29:06.719 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:06 smithi174 systemd[1]: Stopping Ceph osd.0 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:29:06.969 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:06 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[34985]: 2024-02-07T12:29:06.938+0000 7f1609afc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:29:06.970 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:06 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[34985]: 2024-02-07T12:29:06.938+0000 7f1609afc700 -1 osd.0 73 *** Got signal Terminated *** 2024-02-07T12:29:06.970 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:06 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[34985]: 2024-02-07T12:29:06.938+0000 7f1609afc700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:29:07.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 205085 11.14 MB/sec execute 294 sec latency 34.273 ms 2024-02-07T12:29:07.472 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:07 smithi174 ceph-mon[66222]: pgmap v56: 97 pgs: 97 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-02-07T12:29:07.472 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:07 smithi174 ceph-mon[66222]: osd.0 marked itself down and dead 2024-02-07T12:29:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:07 smithi184 ceph-mon[53060]: pgmap v56: 97 pgs: 97 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-02-07T12:29:07.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:07 smithi184 ceph-mon[53060]: osd.0 marked itself down and dead 2024-02-07T12:29:08.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 205371 11.14 MB/sec execute 295 sec latency 700.020 ms 2024-02-07T12:29:08.272 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:07 smithi174 podman[71929]: 2024-02-07 12:29:07.977273344 +0000 UTC m=+1.179770529 container died 948aa062fcd526145557e3f9033cf771d1324597aeec1aef26d99e7b86174a3f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, release=754, io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.expose-services=, version=8, ceph=True, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-02-07T12:29:08.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:08 smithi174 ceph-mon[66222]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:29:08.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:08 smithi174 ceph-mon[66222]: failed to encode map e74 with expected crc 2024-02-07T12:29:08.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:08 smithi174 ceph-mon[66222]: osdmap e74: 8 total, 7 up, 8 in 2024-02-07T12:29:08.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:08 smithi174 ceph-mon[66222]: failed to encode map e74 with expected crc 2024-02-07T12:29:08.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:08 smithi174 ceph-mon[66222]: failed to encode map e74 with expected crc 2024-02-07T12:29:08.525 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:08 smithi174 podman[71929]: 2024-02-07 12:29:08.410917926 +0000 UTC m=+1.613415114 container cleanup 948aa062fcd526145557e3f9033cf771d1324597aeec1aef26d99e7b86174a3f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.buildah.version=1.19.8, name=centos-stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, release=754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc.) 2024-02-07T12:29:08.525 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:08 smithi174 bash[71929]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0 2024-02-07T12:29:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:08 smithi184 ceph-mon[53060]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:29:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:08 smithi184 ceph-mon[53060]: failed to encode map e74 with expected crc 2024-02-07T12:29:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:08 smithi184 ceph-mon[53060]: osdmap e74: 8 total, 7 up, 8 in 2024-02-07T12:29:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:08 smithi184 ceph-mon[53060]: failed to encode map e74 with expected crc 2024-02-07T12:29:08.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:08 smithi184 ceph-mon[53060]: failed to encode map e74 with expected crc 2024-02-07T12:29:08.775 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:08 smithi174 podman[71946]: 2024-02-07 12:29:08.667622892 +0000 UTC m=+0.682090439 container remove 948aa062fcd526145557e3f9033cf771d1324597aeec1aef26d99e7b86174a3f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.buildah.version=1.19.8, ceph=True, distribution-scope=public, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-02-07T12:29:09.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 205743 11.16 MB/sec execute 296 sec latency 1038.406 ms 2024-02-07T12:29:09.277 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:09 smithi174 podman[72033]: 2024-02-07 12:29:09.167947393 +0000 UTC m=+0.032206658 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:29:09.529 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:09 smithi174 podman[72033]: 2024-02-07 12:29:09.306353975 +0000 UTC m=+0.170613225 container create a3e9de49e246295be979798f931ff5e9fd36c409f08df36ff63a55ce042284aa (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-02-07T12:29:09.529 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: pgmap v58: 97 pgs: 13 stale+active+clean, 84 active+clean; 332 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 428 op/s 2024-02-07T12:29:09.529 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.530 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: osdmap e75: 8 total, 7 up, 8 in 2024-02-07T12:29:09.530 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.530 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.530 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.530 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.531 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.531 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.531 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:09 smithi174 ceph-mon[66222]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: pgmap v58: 97 pgs: 13 stale+active+clean, 84 active+clean; 332 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 428 op/s 2024-02-07T12:29:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: osdmap e75: 8 total, 7 up, 8 in 2024-02-07T12:29:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: failed to encode map e75 with expected crc 2024-02-07T12:29:09.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:09 smithi184 ceph-mon[53060]: failed to encode map e75 with expected crc 2024-02-07T12:29:10.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 206335 11.18 MB/sec execute 297 sec latency 31.279 ms 2024-02-07T12:29:10.030 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:09 smithi174 podman[72033]: 2024-02-07 12:29:09.877520387 +0000 UTC m=+0.741779638 container init a3e9de49e246295be979798f931ff5e9fd36c409f08df36ff63a55ce042284aa (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:29:10.031 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:09 smithi174 podman[72033]: 2024-02-07 12:29:09.885624284 +0000 UTC m=+0.749883534 container start a3e9de49e246295be979798f931ff5e9fd36c409f08df36ff63a55ce042284aa (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131) 2024-02-07T12:29:10.282 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:10 smithi174 podman[72033]: 2024-02-07 12:29:10.04208457 +0000 UTC m=+0.906343844 container attach a3e9de49e246295be979798f931ff5e9fd36c409f08df36ff63a55ce042284aa (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD) 2024-02-07T12:29:10.532 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:10 smithi174 podman[72033]: 2024-02-07 12:29:10.328000646 +0000 UTC m=+1.192259905 container died a3e9de49e246295be979798f931ff5e9fd36c409f08df36ff63a55ce042284aa (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, ceph=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:29:11.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 206752 11.18 MB/sec execute 298 sec latency 45.396 ms 2024-02-07T12:29:11.299 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:11 smithi174 podman[72058]: 2024-02-07 12:29:11.269136619 +0000 UTC m=+0.934196649 container remove a3e9de49e246295be979798f931ff5e9fd36c409f08df36ff63a55ce042284aa (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:29:11.551 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:11 smithi174 ceph-mon[66222]: pgmap v60: 97 pgs: 13 stale+active+clean, 84 active+clean; 332 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 MiB/s wr, 253 op/s 2024-02-07T12:29:11.551 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:11 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.0.service: Deactivated successfully. 2024-02-07T12:29:11.551 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:11 smithi174 systemd[1]: Stopped Ceph osd.0 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:29:11.551 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:11 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.0.service: Consumed 3min 40.305s CPU time. 2024-02-07T12:29:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:11 smithi184 ceph-mon[53060]: pgmap v60: 97 pgs: 13 stale+active+clean, 84 active+clean; 332 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 MiB/s wr, 253 op/s 2024-02-07T12:29:11.812 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:11 smithi174 systemd[1]: Starting Ceph osd.0 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:29:12.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 207155 11.16 MB/sec execute 299 sec latency 38.379 ms 2024-02-07T12:29:12.062 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:11 smithi174 podman[72152]: 2024-02-07 12:29:11.839492564 +0000 UTC m=+0.023268473 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:29:12.063 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:11 smithi174 podman[72152]: 2024-02-07 12:29:11.970732865 +0000 UTC m=+0.154508827 container create 0db5efecaa544e1d136b53b8f87075ee3b3ae1f500a137c0d923b7786ceb1357 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2) 2024-02-07T12:29:12.316 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 podman[72152]: 2024-02-07 12:29:12.265732713 +0000 UTC m=+0.449508628 container init 0db5efecaa544e1d136b53b8f87075ee3b3ae1f500a137c0d923b7786ceb1357 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, ceph=True, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-02-07T12:29:12.317 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 podman[72152]: 2024-02-07 12:29:12.272685239 +0000 UTC m=+0.456461145 container start 0db5efecaa544e1d136b53b8f87075ee3b3ae1f500a137c0d923b7786ceb1357 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-02-07T12:29:12.567 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:12 smithi174 ceph-mon[66222]: Health check failed: Degraded data redundancy: 225/1707 objects degraded (13.181%), 29 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:12.568 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 podman[72152]: 2024-02-07 12:29:12.433366563 +0000 UTC m=+0.617142476 container attach 0db5efecaa544e1d136b53b8f87075ee3b3ae1f500a137c0d923b7786ceb1357 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-02-07T12:29:12.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:12 smithi184 ceph-mon[53060]: Health check failed: Degraded data redundancy: 225/1707 objects degraded (13.181%), 29 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:13.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 207474 11.16 MB/sec execute 300 sec latency 66.313 ms 2024-02-07T12:29:13.096 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate[72167]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-07T12:29:13.096 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate[72167]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-02-07T12:29:13.096 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 bash[72152]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-07T12:29:13.096 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 bash[72152]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-02-07T12:29:13.096 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 bash[72152]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 bash[72152]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 bash[72152]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 bash[72152]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 bash[72152]: --> ceph-volume raw activate successful for osd ID: 0 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate[72167]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate[72167]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate[72167]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate[72167]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate[72167]: --> ceph-volume raw activate successful for osd ID: 0 2024-02-07T12:29:13.097 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:12 smithi174 podman[72152]: 2024-02-07 12:29:12.944901049 +0000 UTC m=+1.128676961 container died 0db5efecaa544e1d136b53b8f87075ee3b3ae1f500a137c0d923b7786ceb1357 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD) 2024-02-07T12:29:13.602 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:13 smithi174 ceph-mon[66222]: pgmap v61: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.5 MiB/s wr, 646 op/s; 225/1707 objects degraded (13.181%) 2024-02-07T12:29:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:13 smithi184 ceph-mon[53060]: pgmap v61: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.5 MiB/s wr, 646 op/s; 225/1707 objects degraded (13.181%) 2024-02-07T12:29:13.854 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:13 smithi174 podman[72336]: 2024-02-07 12:29:13.80337623 +0000 UTC m=+0.852889889 container remove 0db5efecaa544e1d136b53b8f87075ee3b3ae1f500a137c0d923b7786ceb1357 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-02-07T12:29:14.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 207923 11.16 MB/sec execute 301 sec latency 45.617 ms 2024-02-07T12:29:14.105 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:14 smithi174 podman[72388]: 2024-02-07 12:29:13.96169305 +0000 UTC m=+0.032175288 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:29:14.106 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:14 smithi174 podman[72388]: 2024-02-07 12:29:14.075064851 +0000 UTC m=+0.145547090 container create aa986beb783de69f1a73e2979e5a7c1173c2deb5834e4933d86e0e3b958c1991 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, ceph=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:29:14.878 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:14 smithi174 ceph-mon[66222]: pgmap v62: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.5 MiB/s wr, 646 op/s; 225/1707 objects degraded (13.181%) 2024-02-07T12:29:14.878 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:14 smithi174 podman[72388]: 2024-02-07 12:29:14.650912143 +0000 UTC m=+0.721394392 container init aa986beb783de69f1a73e2979e5a7c1173c2deb5834e4933d86e0e3b958c1991 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0) 2024-02-07T12:29:14.878 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:14 smithi174 podman[72388]: 2024-02-07 12:29:14.664029603 +0000 UTC m=+0.734511841 container start aa986beb783de69f1a73e2979e5a7c1173c2deb5834e4933d86e0e3b958c1991 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2) 2024-02-07T12:29:14.879 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:14 smithi174 ceph-osd[72406]: -- 172.21.15.174:0/3905899055 <== mon.0 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562c1eaaf6c0 con 0x562c1ea80c00 2024-02-07T12:29:14.879 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:14 smithi174 bash[72388]: aa986beb783de69f1a73e2979e5a7c1173c2deb5834e4933d86e0e3b958c1991 2024-02-07T12:29:14.879 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:14 smithi174 systemd[1]: Started Ceph osd.0 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:29:15.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 208410 11.14 MB/sec execute 302 sec latency 33.265 ms 2024-02-07T12:29:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:14 smithi184 ceph-mon[53060]: pgmap v62: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.5 MiB/s wr, 646 op/s; 225/1707 objects degraded (13.181%) 2024-02-07T12:29:15.389 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:15 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[72402]: 2024-02-07T12:29:15.243+0000 7faea98297c0 -1 Falling back to public interface 2024-02-07T12:29:16.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 208874 11.15 MB/sec execute 303 sec latency 40.132 ms 2024-02-07T12:29:16.157 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:16.157 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:16.157 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:16.924 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:16 smithi174 ceph-mon[66222]: pgmap v63: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.8 KiB/s rd, 4.1 MiB/s wr, 360 op/s; 225/1707 objects degraded (13.181%) 2024-02-07T12:29:17.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 209417 11.19 MB/sec execute 304 sec latency 39.801 ms 2024-02-07T12:29:17.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:16 smithi184 ceph-mon[53060]: pgmap v63: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.8 KiB/s rd, 4.1 MiB/s wr, 360 op/s; 225/1707 objects degraded (13.181%) 2024-02-07T12:29:18.027 INFO:teuthology.orchestra.run.smithi174.stdout: 5 209963 11.21 MB/sec execute 305 sec latency 34.341 ms 2024-02-07T12:29:18.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:18 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 227/1671 objects degraded (13.585%), 29 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:18 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 227/1671 objects degraded (13.585%), 29 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:19.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 210339 11.20 MB/sec execute 306 sec latency 39.910 ms 2024-02-07T12:29:19.203 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:19 smithi174 ceph-mon[66222]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 343 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 628 op/s; 227/1671 objects degraded (13.585%) 2024-02-07T12:29:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:19 smithi184 ceph-mon[53060]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 343 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 628 op/s; 227/1671 objects degraded (13.585%) 2024-02-07T12:29:20.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 210702 11.19 MB/sec execute 307 sec latency 35.287 ms 2024-02-07T12:29:21.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 211002 11.18 MB/sec execute 308 sec latency 36.936 ms 2024-02-07T12:29:21.248 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:21 smithi174 ceph-mon[66222]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 343 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 537 op/s; 227/1671 objects degraded (13.585%) 2024-02-07T12:29:21.249 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:21.249 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:29:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:21 smithi184 ceph-mon[53060]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 343 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 537 op/s; 227/1671 objects degraded (13.585%) 2024-02-07T12:29:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:29:22.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 211383 11.17 MB/sec execute 309 sec latency 73.667 ms 2024-02-07T12:29:22.260 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:22 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[72402]: 2024-02-07T12:29:22.063+0000 7faea98297c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-02-07T12:29:23.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 211657 11.15 MB/sec execute 310 sec latency 69.036 ms 2024-02-07T12:29:23.286 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:23 smithi174 ceph-mon[66222]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 793 op/s; 217/1647 objects degraded (13.175%) 2024-02-07T12:29:23.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:23 smithi184 ceph-mon[53060]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 793 op/s; 217/1647 objects degraded (13.175%) 2024-02-07T12:29:23.547 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:23 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[72402]: 2024-02-07T12:29:23.438+0000 7faea98297c0 -1 osd.0 73 log_to_monitors true 2024-02-07T12:29:24.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 211965 11.13 MB/sec execute 311 sec latency 50.454 ms 2024-02-07T12:29:24.346 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:24 smithi174 ceph-mon[66222]: from='osd.0 [v2:172.21.15.174:6802/1390999985,v1:172.21.15.174:6803/1390999985]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:29:24.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:24.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:24.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:24 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 217/1647 objects degraded (13.175%), 29 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:24.347 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:29:24 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[72402]: 2024-02-07T12:29:24.161+0000 7fae9ac88700 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:29:24.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:24 smithi184 ceph-mon[53060]: from='osd.0 [v2:172.21.15.174:6802/1390999985,v1:172.21.15.174:6803/1390999985]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-02-07T12:29:24.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:24.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:24.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:24 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 217/1647 objects degraded (13.175%), 29 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:25.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 212207 11.11 MB/sec execute 312 sec latency 53.863 ms 2024-02-07T12:29:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 531 op/s; 217/1647 objects degraded (13.175%) 2024-02-07T12:29:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: from='osd.0 [v2:172.21.15.174:6802/1390999985,v1:172.21.15.174:6803/1390999985]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:29:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: failed to encode map e76 with expected crc 2024-02-07T12:29:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: osdmap e76: 8 total, 7 up, 8 in 2024-02-07T12:29:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: from='osd.0 [v2:172.21.15.174:6802/1390999985,v1:172.21.15.174:6803/1390999985]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:29:25.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: failed to encode map e76 with expected crc 2024-02-07T12:29:25.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: failed to encode map e76 with expected crc 2024-02-07T12:29:25.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:25.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:25.507 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 531 op/s; 217/1647 objects degraded (13.175%) 2024-02-07T12:29:25.507 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: from='osd.0 [v2:172.21.15.174:6802/1390999985,v1:172.21.15.174:6803/1390999985]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-02-07T12:29:25.507 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: failed to encode map e76 with expected crc 2024-02-07T12:29:25.508 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: osdmap e76: 8 total, 7 up, 8 in 2024-02-07T12:29:25.508 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: from='osd.0 [v2:172.21.15.174:6802/1390999985,v1:172.21.15.174:6803/1390999985]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:29:25.508 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: failed to encode map e76 with expected crc 2024-02-07T12:29:25.508 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: failed to encode map e76 with expected crc 2024-02-07T12:29:25.508 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:25.508 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:26.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 212254 11.08 MB/sec execute 313 sec latency 864.972 ms 2024-02-07T12:29:26.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:26 smithi174 ceph-mon[66222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:29:26.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:26 smithi174 ceph-mon[66222]: failed to encode map e77 with expected crc 2024-02-07T12:29:26.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:26 smithi174 ceph-mon[66222]: osd.0 [v2:172.21.15.174:6802/1390999985,v1:172.21.15.174:6803/1390999985] boot 2024-02-07T12:29:26.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:26 smithi174 ceph-mon[66222]: osdmap e77: 8 total, 8 up, 8 in 2024-02-07T12:29:26.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:26 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:29:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:26 smithi184 ceph-mon[53060]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:29:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:26 smithi184 ceph-mon[53060]: failed to encode map e77 with expected crc 2024-02-07T12:29:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:26 smithi184 ceph-mon[53060]: osd.0 [v2:172.21.15.174:6802/1390999985,v1:172.21.15.174:6803/1390999985] boot 2024-02-07T12:29:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:26 smithi184 ceph-mon[53060]: osdmap e77: 8 total, 8 up, 8 in 2024-02-07T12:29:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:26 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-02-07T12:29:27.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 212254 11.05 MB/sec execute 314 sec latency 1865.021 ms 2024-02-07T12:29:27.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 403 op/s; 217/1647 objects degraded (13.175%) 2024-02-07T12:29:27.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: osdmap e78: 8 total, 8 up, 8 in 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:27 smithi174 ceph-mon[66222]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 403 op/s; 217/1647 objects degraded (13.175%) 2024-02-07T12:29:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: osdmap e78: 8 total, 8 up, 8 in 2024-02-07T12:29:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:27.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:27 smithi184 ceph-mon[53060]: failed to encode map e78 with expected crc 2024-02-07T12:29:28.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 212465 11.04 MB/sec execute 315 sec latency 2056.580 ms 2024-02-07T12:29:28.331 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:28 smithi174 ceph-mon[66222]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.331 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:28 smithi174 ceph-mon[66222]: osdmap e79: 8 total, 8 up, 8 in 2024-02-07T12:29:28.331 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:28 smithi174 ceph-mon[66222]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.331 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:28 smithi174 ceph-mon[66222]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.331 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:28 smithi174 ceph-mon[66222]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.331 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:28 smithi174 ceph-mon[66222]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.331 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:28 smithi174 ceph-mon[66222]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:28 smithi184 ceph-mon[53060]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:28 smithi184 ceph-mon[53060]: osdmap e79: 8 total, 8 up, 8 in 2024-02-07T12:29:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:28 smithi184 ceph-mon[53060]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:28 smithi184 ceph-mon[53060]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:28 smithi184 ceph-mon[53060]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:28 smithi184 ceph-mon[53060]: failed to encode map e79 with expected crc 2024-02-07T12:29:28.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:28 smithi184 ceph-mon[53060]: failed to encode map e79 with expected crc 2024-02-07T12:29:29.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 212774 11.05 MB/sec execute 316 sec latency 585.815 ms 2024-02-07T12:29:29.367 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: pgmap v73: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 425 op/s; 88/1524 objects degraded (5.774%) 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: osdmap e80: 8 total, 8 up, 8 in 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 88/1524 objects degraded (5.774%), 8 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: osdmap e81: 8 total, 8 up, 8 in 2024-02-07T12:29:29.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:29 smithi174 ceph-mon[66222]: failed to encode map e81 with expected crc 2024-02-07T12:29:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: pgmap v73: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 425 op/s; 88/1524 objects degraded (5.774%) 2024-02-07T12:29:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: osdmap e80: 8 total, 8 up, 8 in 2024-02-07T12:29:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: failed to encode map e80 with expected crc 2024-02-07T12:29:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 88/1524 objects degraded (5.774%), 8 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: osdmap e81: 8 total, 8 up, 8 in 2024-02-07T12:29:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:29 smithi184 ceph-mon[53060]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 213114 11.04 MB/sec execute 317 sec latency 97.683 ms 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: failed to encode map e82 with expected crc 2024-02-07T12:29:30.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: osdmap e82: 8 total, 8 up, 8 in 2024-02-07T12:29:30.376 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:30.376 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:30.376 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:30.376 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:29:30.376 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:30.376 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: failed to encode map e81 with expected crc 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: failed to encode map e82 with expected crc 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: osdmap e82: 8 total, 8 up, 8 in 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:31.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 213392 11.03 MB/sec execute 318 sec latency 271.085 ms 2024-02-07T12:29:31.343 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: failed to encode map e82 with expected crc 2024-02-07T12:29:31.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: failed to encode map e82 with expected crc 2024-02-07T12:29:31.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: failed to encode map e82 with expected crc 2024-02-07T12:29:31.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: pgmap v77: 97 pgs: 23 peering, 3 active+undersized+degraded, 71 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 389 KiB/s wr, 28 op/s; 32/1521 objects degraded (2.104%); 0 B/s, 21 objects/s recovering 2024-02-07T12:29:31.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:31.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:31.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:31.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:31.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:29:31.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:29:31.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-02-07T12:29:31.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:31 smithi174 ceph-mon[66222]: osdmap e83: 8 total, 8 up, 8 in 2024-02-07T12:29:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: failed to encode map e82 with expected crc 2024-02-07T12:29:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: failed to encode map e82 with expected crc 2024-02-07T12:29:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: failed to encode map e82 with expected crc 2024-02-07T12:29:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: pgmap v77: 97 pgs: 23 peering, 3 active+undersized+degraded, 71 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 389 KiB/s wr, 28 op/s; 32/1521 objects degraded (2.104%); 0 B/s, 21 objects/s recovering 2024-02-07T12:29:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:29:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:29:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:29:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-02-07T12:29:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:31 smithi184 ceph-mon[53060]: osdmap e83: 8 total, 8 up, 8 in 2024-02-07T12:29:32.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 213585 11.02 MB/sec execute 319 sec latency 1007.876 ms 2024-02-07T12:29:32.356 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:32 smithi174 ceph-mon[66222]: failed to encode map e83 with expected crc 2024-02-07T12:29:32.356 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:32 smithi174 ceph-mon[66222]: failed to encode map e83 with expected crc 2024-02-07T12:29:32.356 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:32 smithi174 ceph-mon[66222]: failed to encode map e83 with expected crc 2024-02-07T12:29:32.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:32 smithi174 ceph-mon[66222]: failed to encode map e83 with expected crc 2024-02-07T12:29:32.357 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:32 smithi174 ceph-mon[66222]: osdmap e84: 8 total, 8 up, 8 in 2024-02-07T12:29:32.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:32 smithi184 ceph-mon[53060]: failed to encode map e83 with expected crc 2024-02-07T12:29:32.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:32 smithi184 ceph-mon[53060]: failed to encode map e83 with expected crc 2024-02-07T12:29:32.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:32 smithi184 ceph-mon[53060]: failed to encode map e83 with expected crc 2024-02-07T12:29:32.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:32 smithi184 ceph-mon[53060]: failed to encode map e83 with expected crc 2024-02-07T12:29:32.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:32 smithi184 ceph-mon[53060]: osdmap e84: 8 total, 8 up, 8 in 2024-02-07T12:29:33.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 213820 11.00 MB/sec execute 320 sec latency 697.523 ms 2024-02-07T12:29:33.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:33 smithi174 ceph-mon[66222]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:33 smithi174 ceph-mon[66222]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:33 smithi174 ceph-mon[66222]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:33 smithi174 ceph-mon[66222]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:33 smithi174 ceph-mon[66222]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:33 smithi174 ceph-mon[66222]: pgmap v80: 97 pgs: 1 peering, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.5 MiB/s wr, 399 op/s; 25/1641 objects degraded (1.523%); 1.1 MiB/s, 60 objects/s recovering 2024-02-07T12:29:33.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:33 smithi174 ceph-mon[66222]: osdmap e85: 8 total, 8 up, 8 in 2024-02-07T12:29:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:33 smithi184 ceph-mon[53060]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:33 smithi184 ceph-mon[53060]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:33 smithi184 ceph-mon[53060]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:33 smithi184 ceph-mon[53060]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:33 smithi184 ceph-mon[53060]: failed to encode map e84 with expected crc 2024-02-07T12:29:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:33 smithi184 ceph-mon[53060]: pgmap v80: 97 pgs: 1 peering, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.5 MiB/s wr, 399 op/s; 25/1641 objects degraded (1.523%); 1.1 MiB/s, 60 objects/s recovering 2024-02-07T12:29:33.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:33 smithi184 ceph-mon[53060]: osdmap e85: 8 total, 8 up, 8 in 2024-02-07T12:29:33.951 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:29:34.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 214121 10.99 MB/sec execute 321 sec latency 60.324 ms 2024-02-07T12:29:34.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:34 smithi174 ceph-mon[66222]: failed to encode map e85 with expected crc 2024-02-07T12:29:34.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:34 smithi174 ceph-mon[66222]: failed to encode map e85 with expected crc 2024-02-07T12:29:34.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:34 smithi174 ceph-mon[66222]: failed to encode map e85 with expected crc 2024-02-07T12:29:34.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:34 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 25/1641 objects degraded (1.523%), 3 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:34 smithi184 ceph-mon[53060]: failed to encode map e85 with expected crc 2024-02-07T12:29:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:34 smithi184 ceph-mon[53060]: failed to encode map e85 with expected crc 2024-02-07T12:29:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:34 smithi184 ceph-mon[53060]: failed to encode map e85 with expected crc 2024-02-07T12:29:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:34 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 25/1641 objects degraded (1.523%), 3 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (4m) 11s ago 13m 22.7M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (59s) 11s ago 13m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (53s) 48s ago 13m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (4m) 11s ago 13m 51.7M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (7m) 11s ago 7m 38.7M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (7m) 48s ago 7m 13.3M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (3m) 11s ago 15m 502M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (2m) 48s ago 13m 448M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (2m) 11s ago 15m 46.5M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (117s) 48s ago 13m 35.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:29:34.935 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (7m) 11s ago 7m 255M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (7m) 48s ago 7m 82.5M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (4m) 11s ago 13m 16.9M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (3m) 48s ago 12m 18.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (20s) 11s ago 12m 12.0M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (11m) 11s ago 11m 348M 1936M 17.2.0 e1d6a67b021e ca8f3a3e8f3a 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (11m) 11s ago 11m 511M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (10m) 11s ago 10m 393M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (10m) 48s ago 10m 575M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (10m) 48s ago 10m 442M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (9m) 48s ago 9m 343M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (9m) 48s ago 9m 546M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:29:34.936 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (2m) 11s ago 12m 66.7M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:29:35.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 214405 10.97 MB/sec execute 322 sec latency 70.266 ms 2024-02-07T12:29:35.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:35 smithi174 ceph-mon[66222]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:35.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:35 smithi174 ceph-mon[66222]: pgmap v82: 97 pgs: 1 peering, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.7 MiB/s wr, 335 op/s; 25/1641 objects degraded (1.523%); 984 KiB/s, 34 objects/s recovering 2024-02-07T12:29:35.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:35 smithi174 ceph-mon[66222]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:35.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:35.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:29:35.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:35 smithi184 ceph-mon[53060]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:35.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:35 smithi184 ceph-mon[53060]: pgmap v82: 97 pgs: 1 peering, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.7 MiB/s wr, 335 op/s; 25/1641 objects degraded (1.523%); 984 KiB/s, 34 objects/s recovering 2024-02-07T12:29:35.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:35 smithi184 ceph-mon[53060]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:35.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:35.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:29:35.533 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:29:35.534 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:29:35.534 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:29:35.534 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-02-07T12:29:35.534 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 5 2024-02-07T12:29:35.534 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:29:35.534 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:29:36.020 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:29:36.020 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:29:36.020 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "7/23 daemons upgraded", 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:29:36.021 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:29:36.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 214750 10.97 MB/sec execute 323 sec latency 58.595 ms 2024-02-07T12:29:36.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:36 smithi174 ceph-mon[66222]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:36.188 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:36 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/4247215123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:36 smithi184 ceph-mon[53060]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:36 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/4247215123' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:29:36.617 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:29:37.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 215066 10.95 MB/sec execute 324 sec latency 40.033 ms 2024-02-07T12:29:37.196 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:37 smithi174 ceph-mon[66222]: from='client.44181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:37.197 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:37 smithi174 ceph-mon[66222]: pgmap v83: 97 pgs: 1 peering, 96 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.8 KiB/s rd, 3.3 MiB/s wr, 267 op/s; 704 KiB/s, 34 objects/s recovering 2024-02-07T12:29:37.197 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:37 smithi174 ceph-mon[66222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1641 objects degraded (1.523%), 3 pgs degraded) 2024-02-07T12:29:37.197 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:37 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:29:37.197 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:37 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/2394132342' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:29:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:37 smithi184 ceph-mon[53060]: from='client.44181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:29:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:37 smithi184 ceph-mon[53060]: pgmap v83: 97 pgs: 1 peering, 96 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.8 KiB/s rd, 3.3 MiB/s wr, 267 op/s; 704 KiB/s, 34 objects/s recovering 2024-02-07T12:29:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:37 smithi184 ceph-mon[53060]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/1641 objects degraded (1.523%), 3 pgs degraded) 2024-02-07T12:29:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:37 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:29:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:37 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/2394132342' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:29:38.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 215441 10.95 MB/sec execute 325 sec latency 44.643 ms 2024-02-07T12:29:39.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 215848 10.95 MB/sec execute 326 sec latency 50.104 ms 2024-02-07T12:29:39.233 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:39 smithi174 ceph-mon[66222]: pgmap v84: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 494 op/s; 580 KiB/s, 28 objects/s recovering 2024-02-07T12:29:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:39 smithi184 ceph-mon[53060]: pgmap v84: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 494 op/s; 580 KiB/s, 28 objects/s recovering 2024-02-07T12:29:40.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 216229 10.95 MB/sec execute 327 sec latency 42.501 ms 2024-02-07T12:29:41.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 216585 10.96 MB/sec execute 328 sec latency 41.870 ms 2024-02-07T12:29:41.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:41 smithi174 ceph-mon[66222]: pgmap v85: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.0 MiB/s wr, 281 op/s; 0 B/s, 7 objects/s recovering 2024-02-07T12:29:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:41 smithi184 ceph-mon[53060]: pgmap v85: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.0 MiB/s wr, 281 op/s; 0 B/s, 7 objects/s recovering 2024-02-07T12:29:42.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 216991 10.95 MB/sec execute 329 sec latency 56.613 ms 2024-02-07T12:29:43.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 217329 10.95 MB/sec execute 330 sec latency 49.948 ms 2024-02-07T12:29:43.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:43 smithi174 ceph-mon[66222]: pgmap v86: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 482 op/s; 0 B/s, 5 objects/s recovering 2024-02-07T12:29:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:43 smithi184 ceph-mon[53060]: pgmap v86: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 482 op/s; 0 B/s, 5 objects/s recovering 2024-02-07T12:29:44.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 217685 10.94 MB/sec execute 331 sec latency 34.905 ms 2024-02-07T12:29:44.872 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:44 smithi174 ceph-mon[66222]: pgmap v87: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 427 op/s; 0 B/s, 5 objects/s recovering 2024-02-07T12:29:45.028 INFO:teuthology.orchestra.run.smithi174.stdout: 5 218049 10.94 MB/sec execute 332 sec latency 42.862 ms 2024-02-07T12:29:45.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:44 smithi184 ceph-mon[53060]: pgmap v87: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 427 op/s; 0 B/s, 5 objects/s recovering 2024-02-07T12:29:46.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 218420 10.93 MB/sec execute 333 sec latency 48.320 ms 2024-02-07T12:29:46.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:46 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:46.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:46 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:29:46.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:46 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:29:46.347 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:46 smithi174 ceph-mon[66222]: Upgrade: osd.1 is safe to restart 2024-02-07T12:29:46.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:46 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:46.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:46 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:29:46.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:46 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-02-07T12:29:46.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:46 smithi184 ceph-mon[53060]: Upgrade: osd.1 is safe to restart 2024-02-07T12:29:47.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 218734 10.92 MB/sec execute 334 sec latency 46.275 ms 2024-02-07T12:29:47.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:47 smithi174 ceph-mon[66222]: pgmap v88: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 414 op/s; 0 B/s, 4 objects/s recovering 2024-02-07T12:29:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:47 smithi184 ceph-mon[53060]: pgmap v88: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 414 op/s; 0 B/s, 4 objects/s recovering 2024-02-07T12:29:48.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 219114 10.91 MB/sec execute 335 sec latency 36.633 ms 2024-02-07T12:29:49.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 219461 10.91 MB/sec execute 336 sec latency 45.346 ms 2024-02-07T12:29:49.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:49 smithi174 ceph-mon[66222]: pgmap v89: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.9 MiB/s wr, 609 op/s 2024-02-07T12:29:49.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:49 smithi174 ceph-mon[66222]: Upgrade: Updating osd.1 2024-02-07T12:29:49.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:49.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:29:49.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:49.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:49 smithi174 ceph-mon[66222]: Deploying daemon osd.1 on smithi174 2024-02-07T12:29:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:49 smithi184 ceph-mon[53060]: pgmap v89: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.9 MiB/s wr, 609 op/s 2024-02-07T12:29:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:49 smithi184 ceph-mon[53060]: Upgrade: Updating osd.1 2024-02-07T12:29:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-02-07T12:29:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:29:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:49 smithi184 ceph-mon[53060]: Deploying daemon osd.1 on smithi174 2024-02-07T12:29:50.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 219857 10.92 MB/sec execute 337 sec latency 33.176 ms 2024-02-07T12:29:51.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 220256 10.93 MB/sec execute 338 sec latency 52.514 ms 2024-02-07T12:29:51.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:51 smithi174 ceph-mon[66222]: pgmap v90: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 458 op/s 2024-02-07T12:29:51.216 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:51.216 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:29:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:51 smithi184 ceph-mon[53060]: pgmap v90: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 458 op/s 2024-02-07T12:29:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:29:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:29:52.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 220640 10.92 MB/sec execute 339 sec latency 41.892 ms 2024-02-07T12:29:53.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 220942 10.91 MB/sec execute 340 sec latency 40.852 ms 2024-02-07T12:29:53.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:53 smithi174 ceph-mon[66222]: pgmap v91: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 655 op/s 2024-02-07T12:29:53.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:53 smithi184 ceph-mon[53060]: pgmap v91: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 655 op/s 2024-02-07T12:29:53.515 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:53 smithi174 systemd[1]: Stopping Ceph osd.1 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:29:53.788 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:53 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[37853]: 2024-02-07T12:29:53.712+0000 7f442abde700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:29:53.789 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:53 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[37853]: 2024-02-07T12:29:53.712+0000 7f442abde700 -1 osd.1 85 *** Got signal Terminated *** 2024-02-07T12:29:53.789 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:53 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[37853]: 2024-02-07T12:29:53.712+0000 7f442abde700 -1 osd.1 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:29:54.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 221230 10.91 MB/sec execute 341 sec latency 51.169 ms 2024-02-07T12:29:54.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:54 smithi174 ceph-mon[66222]: osd.1 marked itself down and dead 2024-02-07T12:29:54.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:54 smithi184 ceph-mon[53060]: osd.1 marked itself down and dead 2024-02-07T12:29:55.029 INFO:teuthology.orchestra.run.smithi174.stdout: 5 221343 10.88 MB/sec execute 342 sec latency 854.086 ms 2024-02-07T12:29:55.046 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:54 smithi174 podman[74538]: 2024-02-07 12:29:54.792393523 +0000 UTC m=+1.230286231 container died ca8f3a3e8f3ad4d37e71c79cd65c1c6f6b64de1fd317f19d033f97936d8a62a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, version=8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, distribution-scope=public, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.expose-services=, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, name=centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-02-07T12:29:55.325 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:55 smithi174 podman[74538]: 2024-02-07 12:29:55.160500872 +0000 UTC m=+1.598393590 container cleanup ca8f3a3e8f3ad4d37e71c79cd65c1c6f6b64de1fd317f19d033f97936d8a62a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, distribution-scope=public, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, version=8) 2024-02-07T12:29:55.326 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:55 smithi174 bash[74538]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1 2024-02-07T12:29:55.326 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:55 smithi174 ceph-mon[66222]: pgmap v92: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 448 op/s 2024-02-07T12:29:55.326 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:55 smithi174 ceph-mon[66222]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:29:55.326 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:55 smithi174 ceph-mon[66222]: failed to encode map e86 with expected crc 2024-02-07T12:29:55.326 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:55 smithi174 ceph-mon[66222]: osdmap e86: 8 total, 7 up, 8 in 2024-02-07T12:29:55.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:55 smithi184 ceph-mon[53060]: pgmap v92: 97 pgs: 97 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 448 op/s 2024-02-07T12:29:55.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:55 smithi184 ceph-mon[53060]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:29:55.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:55 smithi184 ceph-mon[53060]: failed to encode map e86 with expected crc 2024-02-07T12:29:55.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:55 smithi184 ceph-mon[53060]: osdmap e86: 8 total, 7 up, 8 in 2024-02-07T12:29:55.832 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:55 smithi174 podman[74556]: 2024-02-07 12:29:55.580056538 +0000 UTC m=+0.783068393 container remove ca8f3a3e8f3ad4d37e71c79cd65c1c6f6b64de1fd317f19d033f97936d8a62a7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vcs-type=git, RELEASE=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.openshift.expose-services=, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, version=8, CEPH_POINT_RELEASE=-17.2.0) 2024-02-07T12:29:56.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 221704 10.87 MB/sec execute 343 sec latency 975.712 ms 2024-02-07T12:29:56.083 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:55 smithi174 podman[74634]: 2024-02-07 12:29:55.90389221 +0000 UTC m=+0.026517679 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:29:56.083 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:56 smithi174 podman[74634]: 2024-02-07 12:29:56.030542453 +0000 UTC m=+0.153167930 container create b88b07f0555bf45fbd194197f1423c9bb7fe9f79bae1a4101b502f54468c45e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-02-07T12:29:56.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: osdmap e87: 8 total, 7 up, 8 in 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.337 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:56 smithi174 ceph-mon[66222]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: osdmap e87: 8 total, 7 up, 8 in 2024-02-07T12:29:56.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:56 smithi184 ceph-mon[53060]: failed to encode map e87 with expected crc 2024-02-07T12:29:56.596 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:56 smithi174 podman[74634]: 2024-02-07 12:29:56.434830888 +0000 UTC m=+0.557456354 container init b88b07f0555bf45fbd194197f1423c9bb7fe9f79bae1a4101b502f54468c45e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:29:56.596 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:56 smithi174 podman[74634]: 2024-02-07 12:29:56.445139943 +0000 UTC m=+0.567765397 container start b88b07f0555bf45fbd194197f1423c9bb7fe9f79bae1a4101b502f54468c45e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-02-07T12:29:56.851 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:56 smithi174 podman[74634]: 2024-02-07 12:29:56.602171074 +0000 UTC m=+0.724796531 container attach b88b07f0555bf45fbd194197f1423c9bb7fe9f79bae1a4101b502f54468c45e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.2, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:29:57.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 222210 10.87 MB/sec execute 344 sec latency 33.138 ms 2024-02-07T12:29:57.103 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:56 smithi174 podman[74634]: 2024-02-07 12:29:56.90304741 +0000 UTC m=+1.025672862 container died b88b07f0555bf45fbd194197f1423c9bb7fe9f79bae1a4101b502f54468c45e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:29:57.354 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:57 smithi174 ceph-mon[66222]: pgmap v95: 97 pgs: 3 peering, 13 stale+active+clean, 81 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.0 MiB/s wr, 361 op/s 2024-02-07T12:29:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:57 smithi184 ceph-mon[53060]: pgmap v95: 97 pgs: 3 peering, 13 stale+active+clean, 81 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.0 MiB/s wr, 361 op/s 2024-02-07T12:29:58.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 222607 10.88 MB/sec execute 345 sec latency 29.521 ms 2024-02-07T12:29:58.122 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:57 smithi174 podman[74659]: 2024-02-07 12:29:57.909834671 +0000 UTC m=+0.999563611 container remove b88b07f0555bf45fbd194197f1423c9bb7fe9f79bae1a4101b502f54468c45e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-02-07T12:29:58.123 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:57 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.1.service: Deactivated successfully. 2024-02-07T12:29:58.123 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:57 smithi174 systemd[1]: Stopped Ceph osd.1 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:29:58.123 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:57 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.1.service: Consumed 3min 59.529s CPU time. 2024-02-07T12:29:58.376 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:58 smithi174 ceph-mon[66222]: Health check failed: Degraded data redundancy: 154/1548 objects degraded (9.948%), 21 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:58.376 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:58 smithi174 systemd[1]: Starting Ceph osd.1 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:29:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:58 smithi184 ceph-mon[53060]: Health check failed: Degraded data redundancy: 154/1548 objects degraded (9.948%), 21 pgs degraded (PG_DEGRADED) 2024-02-07T12:29:58.629 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:58 smithi174 podman[74758]: 2024-02-07 12:29:58.448269912 +0000 UTC m=+0.026921017 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:29:58.630 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:58 smithi174 podman[74758]: 2024-02-07 12:29:58.561376339 +0000 UTC m=+0.140027430 container create 37b67af66b2e1b6ea73b9e86916af81fc7f1da26dbec64c75343e300c308b600 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS) 2024-02-07T12:29:58.886 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:58 smithi174 podman[74758]: 2024-02-07 12:29:58.856249482 +0000 UTC m=+0.434900582 container init 37b67af66b2e1b6ea73b9e86916af81fc7f1da26dbec64c75343e300c308b600 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:29:58.886 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:58 smithi174 podman[74758]: 2024-02-07 12:29:58.862309203 +0000 UTC m=+0.440960298 container start 37b67af66b2e1b6ea73b9e86916af81fc7f1da26dbec64c75343e300c308b600 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131) 2024-02-07T12:29:59.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 222991 10.87 MB/sec execute 346 sec latency 45.297 ms 2024-02-07T12:29:59.140 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 podman[74758]: 2024-02-07 12:29:59.015090114 +0000 UTC m=+0.593741211 container attach 37b67af66b2e1b6ea73b9e86916af81fc7f1da26dbec64c75343e300c308b600 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:29:59.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:29:59 smithi174 ceph-mon[66222]: pgmap v96: 97 pgs: 21 active+undersized+degraded, 3 peering, 15 active+undersized, 58 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.9 MiB/s wr, 600 op/s; 154/1548 objects degraded (9.948%) 2024-02-07T12:29:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:29:59 smithi184 ceph-mon[53060]: pgmap v96: 97 pgs: 21 active+undersized+degraded, 3 peering, 15 active+undersized, 58 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.9 MiB/s wr, 600 op/s; 154/1548 objects degraded (9.948%) 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate[74772]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate[74772]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 bash[74758]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 bash[74758]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 bash[74758]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 bash[74758]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 bash[74758]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 bash[74758]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 bash[74758]: --> ceph-volume raw activate successful for osd ID: 1 2024-02-07T12:29:59.645 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate[74772]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-02-07T12:29:59.646 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate[74772]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-07T12:29:59.646 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate[74772]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-02-07T12:29:59.646 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate[74772]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-02-07T12:29:59.646 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate[74772]: --> ceph-volume raw activate successful for osd ID: 1 2024-02-07T12:29:59.646 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:29:59 smithi174 podman[74758]: 2024-02-07 12:29:59.472658615 +0000 UTC m=+1.051309716 container died 37b67af66b2e1b6ea73b9e86916af81fc7f1da26dbec64c75343e300c308b600 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD) 2024-02-07T12:30:00.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 223348 10.88 MB/sec execute 347 sec latency 61.398 ms 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 154/1548 objects degraded (9.948%), 21 pgs degraded 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: [WRN] OSD_DOWN: 1 osds down 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: osd.1 (root=default,host=smithi174) is down 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: [WRN] PG_DEGRADED: Degraded data redundancy: 154/1548 objects degraded (9.948%), 21 pgs degraded 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.d is active+undersized+degraded, acting [4,3] 2024-02-07T12:30:00.407 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-02-07T12:30:00.408 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:00 smithi174 ceph-mon[66222]: pg 4.a is active+undersized+degraded, acting [6,7] 2024-02-07T12:30:00.663 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:00 smithi174 podman[74905]: 2024-02-07 12:30:00.47365931 +0000 UTC m=+0.994497374 container remove 37b67af66b2e1b6ea73b9e86916af81fc7f1da26dbec64c75343e300c308b600 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-02-07T12:30:00.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 154/1548 objects degraded (9.948%), 21 pgs degraded 2024-02-07T12:30:00.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: [WRN] OSD_DOWN: 1 osds down 2024-02-07T12:30:00.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: osd.1 (root=default,host=smithi174) is down 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: [WRN] PG_DEGRADED: Degraded data redundancy: 154/1548 objects degraded (9.948%), 21 pgs degraded 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.d is active+undersized+degraded, acting [4,3] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-02-07T12:30:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:00.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-02-07T12:30:00.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-02-07T12:30:00.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-02-07T12:30:00.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-02-07T12:30:00.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-02-07T12:30:00.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-02-07T12:30:00.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:00 smithi184 ceph-mon[53060]: pg 4.a is active+undersized+degraded, acting [6,7] 2024-02-07T12:30:00.914 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:00 smithi174 podman[74967]: 2024-02-07 12:30:00.645558067 +0000 UTC m=+0.024422732 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:30:01.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 223666 10.88 MB/sec execute 348 sec latency 42.917 ms 2024-02-07T12:30:01.168 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:01 smithi174 podman[74967]: 2024-02-07 12:30:01.157144669 +0000 UTC m=+0.536009326 container create 6e9164f04adbbe31c2b5311c2ed35ae52263cd63f87e68d4e4680a8d913a24bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-02-07T12:30:01.419 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:01 smithi174 ceph-mon[66222]: pgmap v97: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.3 MiB/s wr, 302 op/s; 172/1542 objects degraded (11.154%) 2024-02-07T12:30:01.671 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:01 smithi174 podman[74967]: 2024-02-07 12:30:01.53669859 +0000 UTC m=+0.915563251 container init 6e9164f04adbbe31c2b5311c2ed35ae52263cd63f87e68d4e4680a8d913a24bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-02-07T12:30:01.671 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:01 smithi174 podman[74967]: 2024-02-07 12:30:01.545777578 +0000 UTC m=+0.924642230 container start 6e9164f04adbbe31c2b5311c2ed35ae52263cd63f87e68d4e4680a8d913a24bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.2, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131) 2024-02-07T12:30:01.672 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:01 smithi174 ceph-osd[74986]: -- 172.21.15.174:0/2991117924 <== mon.0 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56376c3af6c0 con 0x56376c380c00 2024-02-07T12:30:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:01 smithi184 ceph-mon[53060]: pgmap v97: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.3 MiB/s wr, 302 op/s; 172/1542 objects degraded (11.154%) 2024-02-07T12:30:01.921 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:01 smithi174 bash[74967]: 6e9164f04adbbe31c2b5311c2ed35ae52263cd63f87e68d4e4680a8d913a24bc 2024-02-07T12:30:01.922 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:01 smithi174 systemd[1]: Started Ceph osd.1 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:30:02.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 224018 10.87 MB/sec execute 349 sec latency 48.258 ms 2024-02-07T12:30:02.172 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:02 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:30:02.111+0000 7f5acb8627c0 -1 Falling back to public interface 2024-02-07T12:30:02.950 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:02.950 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:02.950 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:02.950 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:02 smithi174 ceph-mon[66222]: pgmap v98: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.4 MiB/s wr, 635 op/s; 180/1596 objects degraded (11.278%) 2024-02-07T12:30:03.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 224338 10.86 MB/sec execute 350 sec latency 53.364 ms 2024-02-07T12:30:03.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:03.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:03.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:03.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:02 smithi184 ceph-mon[53060]: pgmap v98: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.4 MiB/s wr, 635 op/s; 180/1596 objects degraded (11.278%) 2024-02-07T12:30:03.820 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:03 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 180/1596 objects degraded (11.278%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:04.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 224606 10.85 MB/sec execute 351 sec latency 58.944 ms 2024-02-07T12:30:04.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:03 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 180/1596 objects degraded (11.278%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:04.840 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:04 smithi174 ceph-mon[66222]: pgmap v99: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 512 op/s; 180/1596 objects degraded (11.278%) 2024-02-07T12:30:05.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 224945 10.85 MB/sec execute 352 sec latency 45.278 ms 2024-02-07T12:30:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:04 smithi184 ceph-mon[53060]: pgmap v99: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 512 op/s; 180/1596 objects degraded (11.278%) 2024-02-07T12:30:06.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 225315 10.84 MB/sec execute 353 sec latency 35.891 ms 2024-02-07T12:30:06.365 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:06.365 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:30:06.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:06.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:30:07.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 225631 10.83 MB/sec execute 354 sec latency 46.768 ms 2024-02-07T12:30:07.110 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:30:07.142 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:07 smithi174 ceph-mon[66222]: pgmap v100: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 464 op/s; 177/1599 objects degraded (11.069%) 2024-02-07T12:30:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:07 smithi184 ceph-mon[53060]: pgmap v100: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 464 op/s; 177/1599 objects degraded (11.069%) 2024-02-07T12:30:08.031 INFO:teuthology.orchestra.run.smithi174.stdout: 5 225903 10.81 MB/sec execute 355 sec latency 61.571 ms 2024-02-07T12:30:08.065 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:30:08.065 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (5m) 44s ago 14m 22.7M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:30:08.065 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (92s) 44s ago 14m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:30:08.065 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (87s) 81s ago 13m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:30:08.065 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (4m) 44s ago 14m 51.7M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:30:08.065 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (8m) 44s ago 8m 38.7M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (8m) 81s ago 8m 13.3M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (4m) 44s ago 15m 502M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (3m) 81s ago 13m 448M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (3m) 44s ago 15m 46.5M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (2m) 81s ago 13m 35.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (8m) 44s ago 8m 255M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (8m) 81s ago 8m 82.5M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (4m) 44s ago 14m 16.9M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (4m) 81s ago 13m 18.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (54s) 44s ago 12m 12.0M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 starting - - - 1936M 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (12m) 44s ago 11m 511M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (11m) 44s ago 11m 393M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (11m) 81s ago 11m 575M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:30:08.066 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (10m) 81s ago 10m 442M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:30:08.067 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (10m) 81s ago 10m 343M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:30:08.067 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (10m) 81s ago 9m 546M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:30:08.067 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (3m) 44s ago 13m 66.7M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:30:08.163 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:08 smithi174 ceph-mon[66222]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:08.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:08 smithi184 ceph-mon[53060]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:08.774 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:30:08.774 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:30:08.774 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:30:08.774 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:30:08.774 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 1 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 5 2024-02-07T12:30:08.775 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:30:08.776 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:30:09.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 226186 10.81 MB/sec execute 356 sec latency 53.777 ms 2024-02-07T12:30:09.193 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:09 smithi174 ceph-mon[66222]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:09.194 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:09 smithi174 ceph-mon[66222]: pgmap v101: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.9 MiB/s wr, 613 op/s; 168/1560 objects degraded (10.769%) 2024-02-07T12:30:09.194 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:09 smithi174 ceph-mon[66222]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:09.194 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:09 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 168/1560 objects degraded (10.769%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:09.194 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:09 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3889404822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:09.194 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:09 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:30:09.030+0000 7f5acb8627c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-02-07T12:30:09.248 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:30:09.248 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:30:09.248 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:30:09.248 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:30:09.248 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:30:09.248 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:30:09.249 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:30:09.249 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:30:09.249 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:30:09.249 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "7/23 daemons upgraded", 2024-02-07T12:30:09.249 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:30:09.249 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:30:09.249 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:30:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:09 smithi184 ceph-mon[53060]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:09 smithi184 ceph-mon[53060]: pgmap v101: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.9 MiB/s wr, 613 op/s; 168/1560 objects degraded (10.769%) 2024-02-07T12:30:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:09 smithi184 ceph-mon[53060]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:09 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 168/1560 objects degraded (10.769%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:09.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:09 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3889404822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:09.821 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 168/1560 objects degraded (10.769%), 23 pgs degraded 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] OSD_DOWN: 1 osds down 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: osd.1 (root=default,host=smithi174) is down 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 168/1560 objects degraded (10.769%), 23 pgs degraded 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.4 is active+undersized+degraded, acting [0,2] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-02-07T12:30:09.822 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-02-07T12:30:09.823 INFO:teuthology.orchestra.run.smithi174.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-02-07T12:30:10.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 226430 10.80 MB/sec execute 357 sec latency 82.468 ms 2024-02-07T12:30:10.249 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:10 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:30:10.199+0000 7f5acb8627c0 -1 osd.1 85 log_to_monitors true 2024-02-07T12:30:10.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:10 smithi174 ceph-mon[66222]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:10.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:10 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3134062409' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:30:10.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:10 smithi184 ceph-mon[53060]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:10.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:10 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3134062409' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:30:11.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 226705 10.79 MB/sec execute 358 sec latency 89.454 ms 2024-02-07T12:30:11.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:11 smithi174 ceph-mon[66222]: pgmap v102: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 451 op/s; 170/1560 objects degraded (10.897%) 2024-02-07T12:30:11.292 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:11 smithi174 ceph-mon[66222]: from='osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:30:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:11 smithi184 ceph-mon[53060]: pgmap v102: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 451 op/s; 170/1560 objects degraded (10.897%) 2024-02-07T12:30:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:11 smithi184 ceph-mon[53060]: from='osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-02-07T12:30:11.796 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:30:11 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:30:11.673+0000 7f5ab8cb9700 -1 osd.1 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:30:12.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 226912 10.78 MB/sec execute 359 sec latency 120.652 ms 2024-02-07T12:30:12.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:12 smithi174 ceph-mon[66222]: from='osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:30:12.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:12 smithi174 ceph-mon[66222]: failed to encode map e88 with expected crc 2024-02-07T12:30:12.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:12 smithi174 ceph-mon[66222]: osdmap e88: 8 total, 7 up, 8 in 2024-02-07T12:30:12.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:12 smithi174 ceph-mon[66222]: from='osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:30:12.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:12 smithi174 ceph-mon[66222]: failed to encode map e88 with expected crc 2024-02-07T12:30:12.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:12.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:12.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:12 smithi174 ceph-mon[66222]: from='osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627]' entity='osd.1' 2024-02-07T12:30:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:12 smithi184 ceph-mon[53060]: from='osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-02-07T12:30:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:12 smithi184 ceph-mon[53060]: failed to encode map e88 with expected crc 2024-02-07T12:30:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:12 smithi184 ceph-mon[53060]: osdmap e88: 8 total, 7 up, 8 in 2024-02-07T12:30:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:12 smithi184 ceph-mon[53060]: from='osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:30:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:12 smithi184 ceph-mon[53060]: failed to encode map e88 with expected crc 2024-02-07T12:30:12.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:12.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:12.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:12 smithi184 ceph-mon[53060]: from='osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627]' entity='osd.1' 2024-02-07T12:30:13.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 227155 10.77 MB/sec execute 360 sec latency 317.258 ms 2024-02-07T12:30:13.328 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:13 smithi174 ceph-mon[66222]: pgmap v104: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.4 MiB/s wr, 455 op/s; 169/1554 objects degraded (10.875%) 2024-02-07T12:30:13.328 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:13 smithi174 ceph-mon[66222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:30:13.329 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:13 smithi174 ceph-mon[66222]: osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627] boot 2024-02-07T12:30:13.329 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:13 smithi174 ceph-mon[66222]: osdmap e89: 8 total, 8 up, 8 in 2024-02-07T12:30:13.329 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:30:13.329 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:13.329 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:13 smithi184 ceph-mon[53060]: pgmap v104: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.4 MiB/s wr, 455 op/s; 169/1554 objects degraded (10.875%) 2024-02-07T12:30:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:13 smithi184 ceph-mon[53060]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:30:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:13 smithi184 ceph-mon[53060]: osd.1 [v2:172.21.15.174:6810/3479127627,v1:172.21.15.174:6811/3479127627] boot 2024-02-07T12:30:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:13 smithi184 ceph-mon[53060]: osdmap e89: 8 total, 8 up, 8 in 2024-02-07T12:30:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-02-07T12:30:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:14.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 227314 10.75 MB/sec execute 361 sec latency 1317.287 ms 2024-02-07T12:30:14.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:14 smithi174 ceph-mon[66222]: failed to encode map e89 with expected crc 2024-02-07T12:30:14.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:14 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:14.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:14 smithi174 ceph-mon[66222]: osdmap e90: 8 total, 8 up, 8 in 2024-02-07T12:30:14.364 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:14 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:14.364 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:14 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:14.364 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:14 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 169/1554 objects degraded (10.875%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:14 smithi184 ceph-mon[53060]: failed to encode map e89 with expected crc 2024-02-07T12:30:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:14 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:14 smithi184 ceph-mon[53060]: osdmap e90: 8 total, 8 up, 8 in 2024-02-07T12:30:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:14 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:14 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:14.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:14 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 169/1554 objects degraded (10.875%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:15.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 227522 10.73 MB/sec execute 362 sec latency 1991.075 ms 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 351 op/s; 169/1554 objects degraded (10.875%) 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: osdmap e91: 8 total, 8 up, 8 in 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e91 with expected crc 2024-02-07T12:30:15.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:15 smithi174 ceph-mon[66222]: failed to encode map e91 with expected crc 2024-02-07T12:30:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e90 with expected crc 2024-02-07T12:30:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 351 op/s; 169/1554 objects degraded (10.875%) 2024-02-07T12:30:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: osdmap e91: 8 total, 8 up, 8 in 2024-02-07T12:30:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e91 with expected crc 2024-02-07T12:30:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:15 smithi184 ceph-mon[53060]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 227942 10.73 MB/sec execute 363 sec latency 47.380 ms 2024-02-07T12:30:16.224 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:16 smithi174 ceph-mon[66222]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.224 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:16 smithi174 ceph-mon[66222]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.224 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:16 smithi174 ceph-mon[66222]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.224 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:16 smithi174 ceph-mon[66222]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.224 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:16 smithi174 ceph-mon[66222]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.224 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:16 smithi174 ceph-mon[66222]: osdmap e92: 8 total, 8 up, 8 in 2024-02-07T12:30:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:16 smithi184 ceph-mon[53060]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:16 smithi184 ceph-mon[53060]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:16 smithi184 ceph-mon[53060]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:16 smithi184 ceph-mon[53060]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:16 smithi184 ceph-mon[53060]: failed to encode map e91 with expected crc 2024-02-07T12:30:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:16 smithi184 ceph-mon[53060]: osdmap e92: 8 total, 8 up, 8 in 2024-02-07T12:30:17.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 228429 10.74 MB/sec execute 364 sec latency 43.464 ms 2024-02-07T12:30:17.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:17 smithi174 ceph-mon[66222]: failed to encode map e92 with expected crc 2024-02-07T12:30:17.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:17 smithi174 ceph-mon[66222]: failed to encode map e92 with expected crc 2024-02-07T12:30:17.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:17 smithi174 ceph-mon[66222]: failed to encode map e92 with expected crc 2024-02-07T12:30:17.241 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:17 smithi174 ceph-mon[66222]: failed to encode map e92 with expected crc 2024-02-07T12:30:17.242 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:17 smithi174 ceph-mon[66222]: pgmap v110: 97 pgs: 1 remapped+peering, 22 active+undersized+degraded, 15 active+undersized, 59 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 403 KiB/s wr, 18 op/s; 156/1539 objects degraded (10.136%) 2024-02-07T12:30:17.242 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:17 smithi174 ceph-mon[66222]: osdmap e93: 8 total, 8 up, 8 in 2024-02-07T12:30:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:17 smithi184 ceph-mon[53060]: failed to encode map e92 with expected crc 2024-02-07T12:30:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:17 smithi184 ceph-mon[53060]: failed to encode map e92 with expected crc 2024-02-07T12:30:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:17 smithi184 ceph-mon[53060]: failed to encode map e92 with expected crc 2024-02-07T12:30:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:17 smithi184 ceph-mon[53060]: failed to encode map e92 with expected crc 2024-02-07T12:30:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:17 smithi184 ceph-mon[53060]: pgmap v110: 97 pgs: 1 remapped+peering, 22 active+undersized+degraded, 15 active+undersized, 59 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 403 KiB/s wr, 18 op/s; 156/1539 objects degraded (10.136%) 2024-02-07T12:30:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:17 smithi184 ceph-mon[53060]: osdmap e93: 8 total, 8 up, 8 in 2024-02-07T12:30:18.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 228706 10.73 MB/sec execute 365 sec latency 784.340 ms 2024-02-07T12:30:18.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:18 smithi174 ceph-mon[66222]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:18 smithi174 ceph-mon[66222]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:18 smithi174 ceph-mon[66222]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:18 smithi174 ceph-mon[66222]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:18 smithi174 ceph-mon[66222]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:18 smithi174 ceph-mon[66222]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:18 smithi174 ceph-mon[66222]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:18 smithi174 ceph-mon[66222]: osdmap e94: 8 total, 8 up, 8 in 2024-02-07T12:30:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:18 smithi184 ceph-mon[53060]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:18 smithi184 ceph-mon[53060]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:18 smithi184 ceph-mon[53060]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:18 smithi184 ceph-mon[53060]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:18 smithi184 ceph-mon[53060]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:18 smithi184 ceph-mon[53060]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:18 smithi184 ceph-mon[53060]: failed to encode map e93 with expected crc 2024-02-07T12:30:18.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:18 smithi184 ceph-mon[53060]: osdmap e94: 8 total, 8 up, 8 in 2024-02-07T12:30:19.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 229043 10.73 MB/sec execute 366 sec latency 579.259 ms 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: pgmap v113: 97 pgs: 1 active+recovery_wait+degraded, 88 active+clean, 1 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 498 op/s; 116/1554 objects degraded (7.465%); 1024 KiB/s, 57 keys/s, 19 objects/s recovering 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:19.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 156/1539 objects degraded (10.136%), 22 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:19.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:19 smithi174 ceph-mon[66222]: osdmap e95: 8 total, 8 up, 8 in 2024-02-07T12:30:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: failed to encode map e94 with expected crc 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: pgmap v113: 97 pgs: 1 active+recovery_wait+degraded, 88 active+clean, 1 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 498 op/s; 116/1554 objects degraded (7.465%); 1024 KiB/s, 57 keys/s, 19 objects/s recovering 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:30:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:30:19.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-02-07T12:30:19.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 156/1539 objects degraded (10.136%), 22 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:19.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:19 smithi184 ceph-mon[53060]: osdmap e95: 8 total, 8 up, 8 in 2024-02-07T12:30:20.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 229499 10.74 MB/sec execute 367 sec latency 36.043 ms 2024-02-07T12:30:20.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:20 smithi174 ceph-mon[66222]: failed to encode map e95 with expected crc 2024-02-07T12:30:20.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:20 smithi174 ceph-mon[66222]: failed to encode map e95 with expected crc 2024-02-07T12:30:20.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:20 smithi174 ceph-mon[66222]: failed to encode map e95 with expected crc 2024-02-07T12:30:20.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:20 smithi174 ceph-mon[66222]: failed to encode map e95 with expected crc 2024-02-07T12:30:20.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:20 smithi174 ceph-mon[66222]: osdmap e96: 8 total, 8 up, 8 in 2024-02-07T12:30:20.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:20.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:30:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:20 smithi184 ceph-mon[53060]: failed to encode map e95 with expected crc 2024-02-07T12:30:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:20 smithi184 ceph-mon[53060]: failed to encode map e95 with expected crc 2024-02-07T12:30:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:20 smithi184 ceph-mon[53060]: failed to encode map e95 with expected crc 2024-02-07T12:30:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:20 smithi184 ceph-mon[53060]: failed to encode map e95 with expected crc 2024-02-07T12:30:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:20 smithi184 ceph-mon[53060]: osdmap e96: 8 total, 8 up, 8 in 2024-02-07T12:30:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:30:21.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 229856 10.73 MB/sec execute 368 sec latency 41.855 ms 2024-02-07T12:30:21.366 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:21 smithi174 ceph-mon[66222]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.367 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:21 smithi174 ceph-mon[66222]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.367 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:21 smithi174 ceph-mon[66222]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.367 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:21 smithi174 ceph-mon[66222]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.367 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:21 smithi174 ceph-mon[66222]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.367 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:21 smithi174 ceph-mon[66222]: pgmap v116: 97 pgs: 1 active+recovery_wait+degraded, 89 active+clean, 1 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.8 MiB/s wr, 512 op/s; 116/1557 objects degraded (7.450%); 1.1 MiB/s, 57 keys/s, 24 objects/s recovering 2024-02-07T12:30:21.367 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:21 smithi174 ceph-mon[66222]: osdmap e97: 8 total, 8 up, 8 in 2024-02-07T12:30:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:21 smithi184 ceph-mon[53060]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:21 smithi184 ceph-mon[53060]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:21 smithi184 ceph-mon[53060]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:21 smithi184 ceph-mon[53060]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:21 smithi184 ceph-mon[53060]: failed to encode map e96 with expected crc 2024-02-07T12:30:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:21 smithi184 ceph-mon[53060]: pgmap v116: 97 pgs: 1 active+recovery_wait+degraded, 89 active+clean, 1 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.8 MiB/s wr, 512 op/s; 116/1557 objects degraded (7.450%); 1.1 MiB/s, 57 keys/s, 24 objects/s recovering 2024-02-07T12:30:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:21 smithi184 ceph-mon[53060]: osdmap e97: 8 total, 8 up, 8 in 2024-02-07T12:30:22.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 230220 10.73 MB/sec execute 369 sec latency 48.366 ms 2024-02-07T12:30:22.402 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:22 smithi174 ceph-mon[66222]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:22 smithi174 ceph-mon[66222]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:22 smithi174 ceph-mon[66222]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:22 smithi174 ceph-mon[66222]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:22 smithi174 ceph-mon[66222]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:22 smithi174 ceph-mon[66222]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:22 smithi184 ceph-mon[53060]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:22 smithi184 ceph-mon[53060]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:22 smithi184 ceph-mon[53060]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:22 smithi184 ceph-mon[53060]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:22 smithi184 ceph-mon[53060]: failed to encode map e97 with expected crc 2024-02-07T12:30:22.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:22 smithi184 ceph-mon[53060]: failed to encode map e97 with expected crc 2024-02-07T12:30:23.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 230544 10.72 MB/sec execute 370 sec latency 46.130 ms 2024-02-07T12:30:23.409 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:23 smithi174 ceph-mon[66222]: pgmap v118: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.3 MiB/s wr, 632 op/s; 51 KiB/s, 32 objects/s recovering 2024-02-07T12:30:23.409 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:23 smithi174 ceph-mon[66222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 116/1557 objects degraded (7.450%), 7 pgs degraded) 2024-02-07T12:30:23.409 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:23 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:30:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:23 smithi184 ceph-mon[53060]: pgmap v118: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.3 MiB/s wr, 632 op/s; 51 KiB/s, 32 objects/s recovering 2024-02-07T12:30:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:23 smithi184 ceph-mon[53060]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 116/1557 objects degraded (7.450%), 7 pgs degraded) 2024-02-07T12:30:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:23 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:30:24.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 230758 10.71 MB/sec execute 371 sec latency 104.032 ms 2024-02-07T12:30:24.954 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:24 smithi174 ceph-mon[66222]: pgmap v119: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 449 op/s; 36 KiB/s, 22 objects/s recovering 2024-02-07T12:30:25.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 231035 10.69 MB/sec execute 372 sec latency 57.849 ms 2024-02-07T12:30:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:24 smithi184 ceph-mon[53060]: pgmap v119: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 449 op/s; 36 KiB/s, 22 objects/s recovering 2024-02-07T12:30:26.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 231304 10.69 MB/sec execute 373 sec latency 56.525 ms 2024-02-07T12:30:27.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 231482 10.67 MB/sec execute 374 sec latency 68.252 ms 2024-02-07T12:30:27.287 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:27 smithi174 ceph-mon[66222]: pgmap v120: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 393 op/s; 30 KiB/s, 19 objects/s recovering 2024-02-07T12:30:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:27 smithi184 ceph-mon[53060]: pgmap v120: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 393 op/s; 30 KiB/s, 19 objects/s recovering 2024-02-07T12:30:28.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 231760 10.67 MB/sec execute 375 sec latency 44.377 ms 2024-02-07T12:30:29.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 232095 10.67 MB/sec execute 376 sec latency 55.569 ms 2024-02-07T12:30:29.349 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:29 smithi174 ceph-mon[66222]: pgmap v121: 97 pgs: 97 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 560 op/s; 0 B/s, 14 objects/s recovering 2024-02-07T12:30:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:29 smithi184 ceph-mon[53060]: pgmap v121: 97 pgs: 97 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 560 op/s; 0 B/s, 14 objects/s recovering 2024-02-07T12:30:30.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 232452 10.66 MB/sec execute 377 sec latency 67.353 ms 2024-02-07T12:30:31.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 232693 10.64 MB/sec execute 378 sec latency 44.358 ms 2024-02-07T12:30:31.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:31 smithi174 ceph-mon[66222]: pgmap v122: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 479 op/s; 0 B/s, 11 objects/s recovering 2024-02-07T12:30:31.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:31 smithi184 ceph-mon[53060]: pgmap v122: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 479 op/s; 0 B/s, 11 objects/s recovering 2024-02-07T12:30:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:32.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 232944 10.64 MB/sec execute 379 sec latency 55.452 ms 2024-02-07T12:30:33.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 233207 10.63 MB/sec execute 380 sec latency 96.709 ms 2024-02-07T12:30:33.197 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:33 smithi174 ceph-mon[66222]: pgmap v123: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.8 MiB/s wr, 604 op/s; 0 B/s, 10 objects/s recovering 2024-02-07T12:30:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:33 smithi184 ceph-mon[53060]: pgmap v123: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.8 MiB/s wr, 604 op/s; 0 B/s, 10 objects/s recovering 2024-02-07T12:30:34.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 233481 10.61 MB/sec execute 381 sec latency 62.662 ms 2024-02-07T12:30:34.230 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:30:34.231 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:34 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:30:34.231 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:34 smithi174 ceph-mon[66222]: Upgrade: osd.2 is safe to restart 2024-02-07T12:30:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:30:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:34 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-02-07T12:30:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:34 smithi184 ceph-mon[53060]: Upgrade: osd.2 is safe to restart 2024-02-07T12:30:35.032 INFO:teuthology.orchestra.run.smithi174.stdout: 5 233726 10.62 MB/sec execute 382 sec latency 59.236 ms 2024-02-07T12:30:35.279 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:35 smithi174 ceph-mon[66222]: pgmap v124: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.5 MiB/s wr, 352 op/s 2024-02-07T12:30:35.280 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:35.280 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:30:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:35 smithi184 ceph-mon[53060]: pgmap v124: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.5 MiB/s wr, 352 op/s 2024-02-07T12:30:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:30:36.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 233930 10.60 MB/sec execute 383 sec latency 72.647 ms 2024-02-07T12:30:37.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 234353 10.60 MB/sec execute 384 sec latency 36.478 ms 2024-02-07T12:30:37.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:37 smithi174 ceph-mon[66222]: Upgrade: Updating osd.2 2024-02-07T12:30:37.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:37 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:37.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:37 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:30:37.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:37 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:30:37.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:37 smithi174 ceph-mon[66222]: Deploying daemon osd.2 on smithi174 2024-02-07T12:30:37.065 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:37 smithi174 ceph-mon[66222]: pgmap v125: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-02-07T12:30:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:37 smithi184 ceph-mon[53060]: Upgrade: Updating osd.2 2024-02-07T12:30:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:37 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:37 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-02-07T12:30:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:37 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:30:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:37 smithi184 ceph-mon[53060]: Deploying daemon osd.2 on smithi174 2024-02-07T12:30:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:37 smithi184 ceph-mon[53060]: pgmap v125: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-02-07T12:30:38.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 234812 10.60 MB/sec execute 385 sec latency 38.174 ms 2024-02-07T12:30:39.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 235083 10.60 MB/sec execute 386 sec latency 45.143 ms 2024-02-07T12:30:39.115 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:39 smithi174 ceph-mon[66222]: pgmap v126: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 502 op/s 2024-02-07T12:30:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:39 smithi184 ceph-mon[53060]: pgmap v126: 97 pgs: 97 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 502 op/s 2024-02-07T12:30:40.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 235357 10.60 MB/sec execute 387 sec latency 80.447 ms 2024-02-07T12:30:40.460 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:30:41.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 235621 10.59 MB/sec execute 388 sec latency 61.343 ms 2024-02-07T12:30:41.164 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:41 smithi174 ceph-mon[66222]: pgmap v127: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 357 op/s 2024-02-07T12:30:41.165 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:41 smithi174 ceph-mon[66222]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:41 smithi184 ceph-mon[53060]: pgmap v127: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 357 op/s 2024-02-07T12:30:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:41 smithi184 ceph-mon[53060]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:41.529 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:30:41.529 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (5m) 30s ago 15m 22.8M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:30:41.529 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (2m) 30s ago 14m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:30:41.529 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (2m) 115s ago 14m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (5m) 30s ago 14m 52.9M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (9m) 30s ago 9m 38.5M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (9m) 115s ago 9m 13.3M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (4m) 30s ago 16m 504M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (3m) 115s ago 14m 448M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (3m) 30s ago 16m 48.6M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (3m) 115s ago 14m 35.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (8m) 30s ago 8m 254M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (8m) 115s ago 8m 82.5M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (5m) 30s ago 14m 17.4M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (4m) 115s ago 14m 18.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (87s) 30s ago 13m 254M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:30:41.530 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (40s) 30s ago 13m 12.7M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:30:41.531 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (12m) 30s ago 12m 565M 1936M 17.2.0 e1d6a67b021e f97415c901ab 2024-02-07T12:30:41.531 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (12m) 30s ago 12m 417M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:30:41.531 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (11m) 115s ago 11m 575M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:30:41.531 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (11m) 115s ago 11m 442M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:30:41.531 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (10m) 115s ago 10m 343M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:30:41.531 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (10m) 115s ago 10m 546M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:30:41.532 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (4m) 30s ago 13m 68.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:30:42.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 235883 10.57 MB/sec execute 389 sec latency 62.903 ms 2024-02-07T12:30:42.047 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:42 smithi174 systemd[1]: Stopping Ceph osd.2 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:30:42.203 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:30:42.203 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:30:42.204 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:30:42.205 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:30:42.205 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-02-07T12:30:42.205 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 6 2024-02-07T12:30:42.205 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:30:42.205 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:30:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:42 smithi174 ceph-mon[66222]: from='client.44215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:42.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:42 smithi174 ceph-mon[66222]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:42.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:42 smithi184 ceph-mon[53060]: from='client.44215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:42.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:42 smithi184 ceph-mon[53060]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:42.553 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:42 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[41648]: 2024-02-07T12:30:42.496+0000 7f8fdc85c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:30:42.553 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:42 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[41648]: 2024-02-07T12:30:42.496+0000 7f8fdc85c700 -1 osd.2 97 *** Got signal Terminated *** 2024-02-07T12:30:42.553 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:42 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[41648]: 2024-02-07T12:30:42.496+0000 7f8fdc85c700 -1 osd.2 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:30:42.747 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:30:42.747 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:30:42.747 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:30:42.747 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:30:42.747 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:30:42.747 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:30:42.747 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:30:42.747 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:30:42.748 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:30:42.748 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "8/23 daemons upgraded", 2024-02-07T12:30:42.748 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:30:42.748 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:30:42.748 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:30:43.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 236121 10.56 MB/sec execute 390 sec latency 60.729 ms 2024-02-07T12:30:43.350 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_WARN 1 osds down 2024-02-07T12:30:43.351 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] OSD_DOWN: 1 osds down 2024-02-07T12:30:43.351 INFO:teuthology.orchestra.run.smithi174.stdout: osd.2 (root=default,host=smithi174) is down 2024-02-07T12:30:43.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:43 smithi174 ceph-mon[66222]: pgmap v128: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 539 op/s 2024-02-07T12:30:43.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:43 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3155883820' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:43.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:43 smithi174 ceph-mon[66222]: osd.2 marked itself down and dead 2024-02-07T12:30:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:43 smithi184 ceph-mon[53060]: pgmap v128: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 539 op/s 2024-02-07T12:30:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:43 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3155883820' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:30:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:43 smithi184 ceph-mon[53060]: osd.2 marked itself down and dead 2024-02-07T12:30:44.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 236142 10.53 MB/sec execute 391 sec latency 902.581 ms 2024-02-07T12:30:44.046 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:43 smithi174 podman[77223]: 2024-02-07 12:30:43.771360524 +0000 UTC m=+1.487975591 container died f97415c901ab8b196869be359cd524ecf53019511b5ccfcf20a63656c57f395e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, distribution-scope=public, vcs-type=git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, RELEASE=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, version=8, architecture=x86_64) 2024-02-07T12:30:44.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:44 smithi174 ceph-mon[66222]: from='client.44225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:44.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:44 smithi174 ceph-mon[66222]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:30:44.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:44 smithi174 ceph-mon[66222]: failed to encode map e98 with expected crc 2024-02-07T12:30:44.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:44 smithi174 ceph-mon[66222]: osdmap e98: 8 total, 7 up, 8 in 2024-02-07T12:30:44.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:44 smithi174 ceph-mon[66222]: failed to encode map e98 with expected crc 2024-02-07T12:30:44.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:44 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/808882198' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:30:44.394 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:44 smithi174 ceph-mon[66222]: osdmap e99: 8 total, 7 up, 8 in 2024-02-07T12:30:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:44 smithi184 ceph-mon[53060]: from='client.44225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:30:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:44 smithi184 ceph-mon[53060]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:30:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:44 smithi184 ceph-mon[53060]: failed to encode map e98 with expected crc 2024-02-07T12:30:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:44 smithi184 ceph-mon[53060]: osdmap e98: 8 total, 7 up, 8 in 2024-02-07T12:30:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:44 smithi184 ceph-mon[53060]: failed to encode map e98 with expected crc 2024-02-07T12:30:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:44 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/808882198' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:30:44.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:44 smithi184 ceph-mon[53060]: osdmap e99: 8 total, 7 up, 8 in 2024-02-07T12:30:44.644 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:44 smithi174 podman[77223]: 2024-02-07 12:30:44.63532462 +0000 UTC m=+2.351939687 container cleanup f97415c901ab8b196869be359cd524ecf53019511b5ccfcf20a63656c57f395e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , release=754, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., architecture=x86_64) 2024-02-07T12:30:44.645 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:44 smithi174 bash[77223]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2 2024-02-07T12:30:45.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 236393 10.52 MB/sec execute 392 sec latency 1055.608 ms 2024-02-07T12:30:45.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: pgmap v130: 97 pgs: 5 stale+active+clean, 92 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 447 op/s 2024-02-07T12:30:45.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.149 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.150 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:45 smithi174 ceph-mon[66222]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.414 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:45 smithi174 podman[77283]: 2024-02-07 12:30:45.207128852 +0000 UTC m=+1.431056268 container remove f97415c901ab8b196869be359cd524ecf53019511b5ccfcf20a63656c57f395e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, name=centos-stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-02-07T12:30:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: pgmap v130: 97 pgs: 5 stale+active+clean, 92 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 447 op/s 2024-02-07T12:30:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:45 smithi184 ceph-mon[53060]: failed to encode map e99 with expected crc 2024-02-07T12:30:45.671 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:45 smithi174 podman[77371]: 2024-02-07 12:30:45.526932305 +0000 UTC m=+0.024351402 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:30:45.672 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:45 smithi174 podman[77371]: 2024-02-07 12:30:45.659383628 +0000 UTC m=+0.156802686 container create e46b61f5e2d1e72b28082da1fc12bb79f234a2f7d1dc865ee6def4526ffd9e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD) 2024-02-07T12:30:46.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 236818 10.52 MB/sec execute 393 sec latency 41.684 ms 2024-02-07T12:30:46.175 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:46 smithi174 podman[77371]: 2024-02-07 12:30:46.065747775 +0000 UTC m=+0.563166845 container init e46b61f5e2d1e72b28082da1fc12bb79f234a2f7d1dc865ee6def4526ffd9e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:30:46.176 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:46 smithi174 podman[77371]: 2024-02-07 12:30:46.075965967 +0000 UTC m=+0.573385031 container start e46b61f5e2d1e72b28082da1fc12bb79f234a2f7d1dc865ee6def4526ffd9e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240131, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2) 2024-02-07T12:30:46.433 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:46 smithi174 ceph-mon[66222]: Health check failed: Degraded data redundancy: 12/1593 objects degraded (0.753%), 1 pg degraded (PG_DEGRADED) 2024-02-07T12:30:46.433 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:46 smithi174 podman[77371]: 2024-02-07 12:30:46.273423638 +0000 UTC m=+0.770842731 container attach e46b61f5e2d1e72b28082da1fc12bb79f234a2f7d1dc865ee6def4526ffd9e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:30:46.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:46 smithi184 ceph-mon[53060]: Health check failed: Degraded data redundancy: 12/1593 objects degraded (0.753%), 1 pg degraded (PG_DEGRADED) 2024-02-07T12:30:46.709 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:46 smithi174 podman[77371]: 2024-02-07 12:30:46.451102485 +0000 UTC m=+0.948521559 container died e46b61f5e2d1e72b28082da1fc12bb79f234a2f7d1dc865ee6def4526ffd9e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:30:47.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 237190 10.53 MB/sec execute 394 sec latency 49.287 ms 2024-02-07T12:30:47.223 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:47 smithi174 ceph-mon[66222]: pgmap v132: 97 pgs: 1 active+undersized, 1 active+undersized+degraded, 5 stale+active+clean, 90 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.4 MiB/s wr, 323 op/s; 12/1593 objects degraded (0.753%) 2024-02-07T12:30:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:47 smithi184 ceph-mon[53060]: pgmap v132: 97 pgs: 1 active+undersized, 1 active+undersized+degraded, 5 stale+active+clean, 90 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.4 MiB/s wr, 323 op/s; 12/1593 objects degraded (0.753%) 2024-02-07T12:30:47.734 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:47 smithi174 podman[77396]: 2024-02-07 12:30:47.548774491 +0000 UTC m=+1.090312675 container remove e46b61f5e2d1e72b28082da1fc12bb79f234a2f7d1dc865ee6def4526ffd9e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-02-07T12:30:47.735 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:47 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.2.service: Deactivated successfully. 2024-02-07T12:30:47.735 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:47 smithi174 systemd[1]: Stopped Ceph osd.2 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:30:47.735 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:47 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.2.service: Consumed 3min 14.918s CPU time. 2024-02-07T12:30:47.987 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:47 smithi174 systemd[1]: Starting Ceph osd.2 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:30:48.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 237485 10.52 MB/sec execute 395 sec latency 56.624 ms 2024-02-07T12:30:48.245 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:48 smithi174 podman[77495]: 2024-02-07 12:30:48.096453392 +0000 UTC m=+0.021976301 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:30:48.246 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:48 smithi174 podman[77495]: 2024-02-07 12:30:48.223673809 +0000 UTC m=+0.149196706 container create 0bf8800e905760b86bd23679f76580eefe15a33369139352936ac03364e6df45 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240131, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-02-07T12:30:48.769 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:48 smithi174 podman[77495]: 2024-02-07 12:30:48.60478554 +0000 UTC m=+0.530308432 container init 0bf8800e905760b86bd23679f76580eefe15a33369139352936ac03364e6df45 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-02-07T12:30:48.769 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:48 smithi174 podman[77495]: 2024-02-07 12:30:48.611908846 +0000 UTC m=+0.537431739 container start 0bf8800e905760b86bd23679f76580eefe15a33369139352936ac03364e6df45 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:30:48.769 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:48 smithi174 podman[77495]: 2024-02-07 12:30:48.767413114 +0000 UTC m=+0.692936012 container attach 0bf8800e905760b86bd23679f76580eefe15a33369139352936ac03364e6df45 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-02-07T12:30:49.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 237888 10.52 MB/sec execute 396 sec latency 50.414 ms 2024-02-07T12:30:49.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:49 smithi174 ceph-mon[66222]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 509 op/s; 162/1575 objects degraded (10.286%) 2024-02-07T12:30:49.271 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate[77508]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-07T12:30:49.271 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate[77508]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-02-07T12:30:49.271 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 bash[77495]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-07T12:30:49.271 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 bash[77495]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-02-07T12:30:49.271 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 bash[77495]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 bash[77495]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 bash[77495]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 bash[77495]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 bash[77495]: --> ceph-volume raw activate successful for osd ID: 2 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate[77508]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate[77508]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate[77508]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate[77508]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate[77508]: --> ceph-volume raw activate successful for osd ID: 2 2024-02-07T12:30:49.272 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:49 smithi174 podman[77495]: 2024-02-07 12:30:49.257227321 +0000 UTC m=+1.182750222 container died 0bf8800e905760b86bd23679f76580eefe15a33369139352936ac03364e6df45 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS) 2024-02-07T12:30:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:49 smithi184 ceph-mon[53060]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 509 op/s; 162/1575 objects degraded (10.286%) 2024-02-07T12:30:50.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 238307 10.52 MB/sec execute 397 sec latency 49.227 ms 2024-02-07T12:30:50.786 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:50 smithi174 podman[77630]: 2024-02-07 12:30:50.655195962 +0000 UTC m=+1.388620670 container remove 0bf8800e905760b86bd23679f76580eefe15a33369139352936ac03364e6df45 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-activate, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131) 2024-02-07T12:30:51.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 238643 10.52 MB/sec execute 398 sec latency 46.252 ms 2024-02-07T12:30:51.044 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:50 smithi174 podman[77687]: 2024-02-07 12:30:50.857627228 +0000 UTC m=+0.042218993 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:30:51.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:51 smithi174 ceph-mon[66222]: pgmap v134: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 348 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.5 MiB/s wr, 237 op/s; 164/1614 objects degraded (10.161%) 2024-02-07T12:30:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:30:51.297 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:51 smithi174 podman[77687]: 2024-02-07 12:30:51.062390655 +0000 UTC m=+0.246982415 container create 5dcb68e59527b2940793285ccbdd41be528954610a303ff6e45453bf731f568f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131) 2024-02-07T12:30:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:51 smithi184 ceph-mon[53060]: pgmap v134: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 348 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.5 MiB/s wr, 237 op/s; 164/1614 objects degraded (10.161%) 2024-02-07T12:30:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:30:51.548 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:51 smithi174 podman[77687]: 2024-02-07 12:30:51.455214488 +0000 UTC m=+0.639806249 container init 5dcb68e59527b2940793285ccbdd41be528954610a303ff6e45453bf731f568f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131) 2024-02-07T12:30:51.548 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:51 smithi174 podman[77687]: 2024-02-07 12:30:51.466086882 +0000 UTC m=+0.650678649 container start 5dcb68e59527b2940793285ccbdd41be528954610a303ff6e45453bf731f568f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-02-07T12:30:51.548 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:51 smithi174 ceph-osd[77705]: -- 172.21.15.174:0/2589617382 <== mon.0 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557a779ab6c0 con 0x557a7797cc00 2024-02-07T12:30:51.810 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:51 smithi174 bash[77687]: 5dcb68e59527b2940793285ccbdd41be528954610a303ff6e45453bf731f568f 2024-02-07T12:30:51.810 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:51 smithi174 systemd[1]: Started Ceph osd.2 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:30:52.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 239117 10.53 MB/sec execute 399 sec latency 82.875 ms 2024-02-07T12:30:52.084 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:52 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[77701]: 2024-02-07T12:30:52.047+0000 7fa89e8557c0 -1 Falling back to public interface 2024-02-07T12:30:52.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:52.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:52.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:52.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:52 smithi174 ceph-mon[66222]: pgmap v135: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 503 op/s; 160/1659 objects degraded (9.644%) 2024-02-07T12:30:53.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 239617 10.53 MB/sec execute 400 sec latency 54.325 ms 2024-02-07T12:30:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:30:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:30:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:52 smithi184 ceph-mon[53060]: pgmap v135: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 503 op/s; 160/1659 objects degraded (9.644%) 2024-02-07T12:30:53.856 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:53 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 160/1659 objects degraded (9.644%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:54.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 239940 10.52 MB/sec execute 401 sec latency 37.813 ms 2024-02-07T12:30:54.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:53 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 160/1659 objects degraded (9.644%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:54.868 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:54 smithi174 ceph-mon[66222]: pgmap v136: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 449 op/s; 160/1659 objects degraded (9.644%) 2024-02-07T12:30:55.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 240351 10.53 MB/sec execute 402 sec latency 59.585 ms 2024-02-07T12:30:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:54 smithi184 ceph-mon[53060]: pgmap v136: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 449 op/s; 160/1659 objects degraded (9.644%) 2024-02-07T12:30:56.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 240751 10.53 MB/sec execute 403 sec latency 64.819 ms 2024-02-07T12:30:57.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 241095 10.52 MB/sec execute 404 sec latency 57.421 ms 2024-02-07T12:30:57.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:57 smithi174 ceph-mon[66222]: pgmap v137: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 394 op/s; 158/1656 objects degraded (9.541%) 2024-02-07T12:30:57.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:57 smithi184 ceph-mon[53060]: pgmap v137: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 394 op/s; 158/1656 objects degraded (9.541%) 2024-02-07T12:30:58.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 241449 10.52 MB/sec execute 405 sec latency 65.522 ms 2024-02-07T12:30:58.202 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:58 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 154/1614 objects degraded (9.542%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:58.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:58 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 154/1614 objects degraded (9.542%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:30:58.971 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:30:58 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[77701]: 2024-02-07T12:30:58.945+0000 7fa89e8557c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-02-07T12:30:59.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 241840 10.52 MB/sec execute 406 sec latency 41.351 ms 2024-02-07T12:30:59.223 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:30:59 smithi174 ceph-mon[66222]: pgmap v138: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 623 op/s; 154/1614 objects degraded (9.542%) 2024-02-07T12:30:59.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:30:59 smithi184 ceph-mon[53060]: pgmap v138: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 623 op/s; 154/1614 objects degraded (9.542%) 2024-02-07T12:31:00.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 242015 10.51 MB/sec execute 407 sec latency 111.074 ms 2024-02-07T12:31:00.248 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:31:00 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[77701]: 2024-02-07T12:31:00.214+0000 7fa89e8557c0 -1 osd.2 97 log_to_monitors true 2024-02-07T12:31:01.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 242244 10.50 MB/sec execute 408 sec latency 32.521 ms 2024-02-07T12:31:01.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:01 smithi174 ceph-mon[66222]: pgmap v139: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 501 op/s; 156/1626 objects degraded (9.594%) 2024-02-07T12:31:01.283 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:01 smithi174 ceph-mon[66222]: from='osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:31:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:01 smithi184 ceph-mon[53060]: pgmap v139: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 501 op/s; 156/1626 objects degraded (9.594%) 2024-02-07T12:31:01.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:01 smithi184 ceph-mon[53060]: from='osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-02-07T12:31:01.793 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:31:01 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[77701]: 2024-02-07T12:31:01.676+0000 7fa88bcac700 -1 osd.2 97 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:31:02.033 INFO:teuthology.orchestra.run.smithi174.stdout: 5 242483 10.50 MB/sec execute 409 sec latency 168.514 ms 2024-02-07T12:31:02.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:02 smithi174 ceph-mon[66222]: from='osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:31:02.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:02 smithi174 ceph-mon[66222]: failed to encode map e100 with expected crc 2024-02-07T12:31:02.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:02 smithi174 ceph-mon[66222]: osdmap e100: 8 total, 7 up, 8 in 2024-02-07T12:31:02.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:02 smithi174 ceph-mon[66222]: from='osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:31:02.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:02 smithi174 ceph-mon[66222]: from='osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967]' entity='osd.2' 2024-02-07T12:31:02.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:02.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:02 smithi184 ceph-mon[53060]: from='osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-02-07T12:31:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:02 smithi184 ceph-mon[53060]: failed to encode map e100 with expected crc 2024-02-07T12:31:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:02 smithi184 ceph-mon[53060]: osdmap e100: 8 total, 7 up, 8 in 2024-02-07T12:31:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:02 smithi184 ceph-mon[53060]: from='osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:31:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:02 smithi184 ceph-mon[53060]: from='osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967]' entity='osd.2' 2024-02-07T12:31:02.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:02.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:03.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 242707 10.48 MB/sec execute 410 sec latency 254.429 ms 2024-02-07T12:31:03.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:03 smithi174 ceph-mon[66222]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 539 op/s; 151/1566 objects degraded (9.642%) 2024-02-07T12:31:03.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:03 smithi174 ceph-mon[66222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:31:03.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:03 smithi174 ceph-mon[66222]: failed to encode map e101 with expected crc 2024-02-07T12:31:03.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:03 smithi174 ceph-mon[66222]: osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967] boot 2024-02-07T12:31:03.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:03 smithi174 ceph-mon[66222]: osdmap e101: 8 total, 8 up, 8 in 2024-02-07T12:31:03.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:03 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:31:03.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:03 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:03.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:03 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:03 smithi184 ceph-mon[53060]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 539 op/s; 151/1566 objects degraded (9.642%) 2024-02-07T12:31:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:03 smithi184 ceph-mon[53060]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:31:03.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:03 smithi184 ceph-mon[53060]: failed to encode map e101 with expected crc 2024-02-07T12:31:03.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:03 smithi184 ceph-mon[53060]: osd.2 [v2:172.21.15.174:6818/1571775967,v1:172.21.15.174:6819/1571775967] boot 2024-02-07T12:31:03.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:03 smithi184 ceph-mon[53060]: osdmap e101: 8 total, 8 up, 8 in 2024-02-07T12:31:03.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:03 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-02-07T12:31:03.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:03 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:03.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:03 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:04.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 242839 10.46 MB/sec execute 411 sec latency 1254.460 ms 2024-02-07T12:31:04.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: osdmap e102: 8 total, 8 up, 8 in 2024-02-07T12:31:04.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 151/1566 objects degraded (9.642%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:04.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:04 smithi184 ceph-mon[53060]: pgmap v144: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.0 MiB/s wr, 388 op/s; 151/1566 objects degraded (9.642%) 2024-02-07T12:31:04.952 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.952 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: osdmap e102: 8 total, 8 up, 8 in 2024-02-07T12:31:04.952 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.952 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.952 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.952 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.952 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.953 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: failed to encode map e102 with expected crc 2024-02-07T12:31:04.953 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 151/1566 objects degraded (9.642%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:04.953 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:04 smithi174 ceph-mon[66222]: pgmap v144: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.0 MiB/s wr, 388 op/s; 151/1566 objects degraded (9.642%) 2024-02-07T12:31:05.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 242938 10.44 MB/sec execute 412 sec latency 2118.732 ms 2024-02-07T12:31:05.709 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:05 smithi174 ceph-mon[66222]: failed to encode map e103 with expected crc 2024-02-07T12:31:05.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:05 smithi174 ceph-mon[66222]: failed to encode map e103 with expected crc 2024-02-07T12:31:05.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:05 smithi174 ceph-mon[66222]: osdmap e103: 8 total, 8 up, 8 in 2024-02-07T12:31:05.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:05 smithi174 ceph-mon[66222]: failed to encode map e103 with expected crc 2024-02-07T12:31:05.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:05 smithi174 ceph-mon[66222]: failed to encode map e103 with expected crc 2024-02-07T12:31:05.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:05 smithi174 ceph-mon[66222]: failed to encode map e103 with expected crc 2024-02-07T12:31:05.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:05.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:06.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 243277 10.43 MB/sec execute 413 sec latency 56.470 ms 2024-02-07T12:31:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:05 smithi184 ceph-mon[53060]: failed to encode map e103 with expected crc 2024-02-07T12:31:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:05 smithi184 ceph-mon[53060]: failed to encode map e103 with expected crc 2024-02-07T12:31:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:05 smithi184 ceph-mon[53060]: osdmap e103: 8 total, 8 up, 8 in 2024-02-07T12:31:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:05 smithi184 ceph-mon[53060]: failed to encode map e103 with expected crc 2024-02-07T12:31:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:05 smithi184 ceph-mon[53060]: failed to encode map e103 with expected crc 2024-02-07T12:31:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:05 smithi184 ceph-mon[53060]: failed to encode map e103 with expected crc 2024-02-07T12:31:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:06.976 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:06 smithi174 ceph-mon[66222]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:31:06.976 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:06 smithi174 ceph-mon[66222]: failed to encode map e104 with expected crc 2024-02-07T12:31:06.976 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:06 smithi174 ceph-mon[66222]: failed to encode map e104 with expected crc 2024-02-07T12:31:06.976 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:06 smithi174 ceph-mon[66222]: failed to encode map e104 with expected crc 2024-02-07T12:31:06.976 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:06 smithi174 ceph-mon[66222]: failed to encode map e104 with expected crc 2024-02-07T12:31:06.976 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:06 smithi174 ceph-mon[66222]: failed to encode map e104 with expected crc 2024-02-07T12:31:06.976 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:06 smithi174 ceph-mon[66222]: failed to encode map e104 with expected crc 2024-02-07T12:31:06.976 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:06 smithi174 ceph-mon[66222]: pgmap v147: 97 pgs: 1 remapped+peering, 5 active+undersized, 19 active+undersized+degraded, 72 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 681 KiB/s wr, 28 op/s; 137/1539 objects degraded (8.902%) 2024-02-07T12:31:07.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 243566 10.43 MB/sec execute 414 sec latency 69.353 ms 2024-02-07T12:31:07.204 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:06 smithi184 ceph-mon[53060]: osdmap e104: 8 total, 8 up, 8 in 2024-02-07T12:31:07.204 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:06 smithi184 ceph-mon[53060]: failed to encode map e104 with expected crc 2024-02-07T12:31:07.204 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:06 smithi184 ceph-mon[53060]: failed to encode map e104 with expected crc 2024-02-07T12:31:07.204 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:06 smithi184 ceph-mon[53060]: failed to encode map e104 with expected crc 2024-02-07T12:31:07.204 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:06 smithi184 ceph-mon[53060]: failed to encode map e104 with expected crc 2024-02-07T12:31:07.205 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:06 smithi184 ceph-mon[53060]: failed to encode map e104 with expected crc 2024-02-07T12:31:07.205 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:06 smithi184 ceph-mon[53060]: failed to encode map e104 with expected crc 2024-02-07T12:31:07.205 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:06 smithi184 ceph-mon[53060]: pgmap v147: 97 pgs: 1 remapped+peering, 5 active+undersized, 19 active+undersized+degraded, 72 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 681 KiB/s wr, 28 op/s; 137/1539 objects degraded (8.902%) 2024-02-07T12:31:08.002 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:07 smithi174 ceph-mon[66222]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.002 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:07 smithi174 ceph-mon[66222]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.002 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:07 smithi174 ceph-mon[66222]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:31:08.002 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:07 smithi174 ceph-mon[66222]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.002 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:07 smithi174 ceph-mon[66222]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.002 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:07 smithi174 ceph-mon[66222]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.003 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:07 smithi174 ceph-mon[66222]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.003 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:07 smithi174 ceph-mon[66222]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 243824 10.42 MB/sec execute 415 sec latency 593.380 ms 2024-02-07T12:31:08.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:07 smithi184 ceph-mon[53060]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:07 smithi184 ceph-mon[53060]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:07 smithi184 ceph-mon[53060]: osdmap e105: 8 total, 8 up, 8 in 2024-02-07T12:31:08.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:07 smithi184 ceph-mon[53060]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:07 smithi184 ceph-mon[53060]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:07 smithi184 ceph-mon[53060]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:07 smithi184 ceph-mon[53060]: failed to encode map e105 with expected crc 2024-02-07T12:31:08.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:07 smithi184 ceph-mon[53060]: failed to encode map e105 with expected crc 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: osdmap e106: 8 total, 8 up, 8 in 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: pgmap v150: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.3 MiB/s wr, 394 op/s; 101/1509 objects degraded (6.693%); 2.2 MiB/s, 17 objects/s recovering 2024-02-07T12:31:09.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 137/1539 objects degraded (8.902%), 19 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:09.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: osdmap e107: 8 total, 8 up, 8 in 2024-02-07T12:31:09.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:08 smithi174 ceph-mon[66222]: failed to encode map e107 with expected crc 2024-02-07T12:31:09.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 244111 10.41 MB/sec execute 416 sec latency 994.474 ms 2024-02-07T12:31:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: osdmap e106: 8 total, 8 up, 8 in 2024-02-07T12:31:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: failed to encode map e106 with expected crc 2024-02-07T12:31:09.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: pgmap v150: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.3 MiB/s wr, 394 op/s; 101/1509 objects degraded (6.693%); 2.2 MiB/s, 17 objects/s recovering 2024-02-07T12:31:09.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 137/1539 objects degraded (8.902%), 19 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:09.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: osdmap e107: 8 total, 8 up, 8 in 2024-02-07T12:31:09.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:08 smithi184 ceph-mon[53060]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 244354 10.40 MB/sec execute 417 sec latency 535.401 ms 2024-02-07T12:31:10.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:10.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:10.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:10.063 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:31:10.063 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:10.063 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:10.063 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:09 smithi174 ceph-mon[66222]: osdmap e108: 8 total, 8 up, 8 in 2024-02-07T12:31:10.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: failed to encode map e107 with expected crc 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:10.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:09 smithi184 ceph-mon[53060]: osdmap e108: 8 total, 8 up, 8 in 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: failed to encode map e108 with expected crc 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: failed to encode map e108 with expected crc 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: failed to encode map e108 with expected crc 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: failed to encode map e108 with expected crc 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: failed to encode map e108 with expected crc 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:31:10.825 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:31:10.826 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-02-07T12:31:10.826 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: pgmap v153: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 399 op/s; 101/1512 objects degraded (6.680%); 2.2 MiB/s, 23 objects/s recovering 2024-02-07T12:31:10.826 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:10 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:11.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 244549 10.39 MB/sec execute 418 sec latency 46.523 ms 2024-02-07T12:31:11.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: failed to encode map e108 with expected crc 2024-02-07T12:31:11.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: failed to encode map e108 with expected crc 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: failed to encode map e108 with expected crc 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: failed to encode map e108 with expected crc 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: failed to encode map e108 with expected crc 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: pgmap v153: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 399 op/s; 101/1512 objects degraded (6.680%); 2.2 MiB/s, 23 objects/s recovering 2024-02-07T12:31:11.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:10 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:12.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 244769 10.39 MB/sec execute 419 sec latency 66.055 ms 2024-02-07T12:31:12.377 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:12 smithi174 ceph-mon[66222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 101/1512 objects degraded (6.680%), 6 pgs degraded) 2024-02-07T12:31:12.377 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:12 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:31:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:12 smithi184 ceph-mon[53060]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 101/1512 objects degraded (6.680%), 6 pgs degraded) 2024-02-07T12:31:12.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:12 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:31:13.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 244983 10.38 MB/sec execute 420 sec latency 68.369 ms 2024-02-07T12:31:13.186 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:13 smithi174 ceph-mon[66222]: pgmap v154: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.7 MiB/s wr, 354 op/s; 840 KiB/s, 23 objects/s recovering 2024-02-07T12:31:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:13 smithi184 ceph-mon[53060]: pgmap v154: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.7 MiB/s wr, 354 op/s; 840 KiB/s, 23 objects/s recovering 2024-02-07T12:31:13.872 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:31:14.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 245327 10.37 MB/sec execute 421 sec latency 58.063 ms 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (6m) 13s ago 15m 22.8M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (2m) 13s ago 15m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (2m) 2m ago 14m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (6m) 13s ago 15m 51.7M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (9m) 13s ago 9m 38.5M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (9m) 2m ago 9m 13.3M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (5m) 13s ago 16m 506M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (4m) 2m ago 14m 448M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (4m) 13s ago 16m 48.3M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:31:14.890 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (3m) 2m ago 14m 35.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (9m) 13s ago 9m 253M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (9m) 2m ago 9m 82.5M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (5m) 13s ago 15m 17.5M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (5m) 2m ago 14m 18.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (2m) 13s ago 13m 310M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (74s) 13s ago 13m 255M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (24s) 13s ago 13m 12.7M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (12m) 13s ago 12m 435M 1936M 17.2.0 e1d6a67b021e 01f07f2e9a5c 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (12m) 2m ago 12m 575M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (11m) 2m ago 11m 442M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (11m) 2m ago 11m 343M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (11m) 2m ago 11m 546M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:31:14.891 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (4m) 13s ago 14m 68.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:31:14.965 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:14 smithi174 ceph-mon[66222]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:14.966 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:14 smithi174 ceph-mon[66222]: pgmap v155: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.1 MiB/s wr, 298 op/s; 707 KiB/s, 20 objects/s recovering 2024-02-07T12:31:14.966 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:14 smithi174 ceph-mon[66222]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:15.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 245611 10.36 MB/sec execute 422 sec latency 49.134 ms 2024-02-07T12:31:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:14 smithi184 ceph-mon[53060]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:14 smithi184 ceph-mon[53060]: pgmap v155: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.1 MiB/s wr, 298 op/s; 707 KiB/s, 20 objects/s recovering 2024-02-07T12:31:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:14 smithi184 ceph-mon[53060]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:15.493 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:31:15.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:31:15.495 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:31:15.495 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:31:15.495 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-02-07T12:31:15.495 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 7 2024-02-07T12:31:15.495 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:31:15.495 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "9/23 daemons upgraded", 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:31:15.996 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:31:15.997 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:15 smithi174 ceph-mon[66222]: from='client.44237 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:15.997 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:15 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/2261617816' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:16.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 245831 10.35 MB/sec execute 423 sec latency 50.561 ms 2024-02-07T12:31:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:15 smithi184 ceph-mon[53060]: from='client.44237 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:16.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:15 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/2261617816' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:16.696 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:31:17.013 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:16 smithi174 ceph-mon[66222]: from='client.44243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:17.013 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:16 smithi174 ceph-mon[66222]: pgmap v156: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.7 MiB/s wr, 246 op/s; 553 KiB/s, 15 objects/s recovering 2024-02-07T12:31:17.013 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:16 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/890789257' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:31:17.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 246104 10.35 MB/sec execute 424 sec latency 49.871 ms 2024-02-07T12:31:17.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:16 smithi184 ceph-mon[53060]: from='client.44243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:17.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:16 smithi184 ceph-mon[53060]: pgmap v156: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.7 MiB/s wr, 246 op/s; 553 KiB/s, 15 objects/s recovering 2024-02-07T12:31:17.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:16 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/890789257' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:31:18.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 246649 10.36 MB/sec execute 425 sec latency 30.358 ms 2024-02-07T12:31:19.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 246980 10.35 MB/sec execute 426 sec latency 35.364 ms 2024-02-07T12:31:19.162 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:19 smithi174 ceph-mon[66222]: pgmap v157: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 409 op/s; 477 KiB/s, 13 objects/s recovering 2024-02-07T12:31:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:19 smithi184 ceph-mon[53060]: pgmap v157: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 409 op/s; 477 KiB/s, 13 objects/s recovering 2024-02-07T12:31:20.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 247361 10.35 MB/sec execute 427 sec latency 49.099 ms 2024-02-07T12:31:21.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 247746 10.35 MB/sec execute 428 sec latency 49.069 ms 2024-02-07T12:31:21.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:21 smithi174 ceph-mon[66222]: pgmap v158: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 372 op/s; 427 KiB/s, 10 objects/s recovering 2024-02-07T12:31:21.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:21.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:21.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:21 smithi184 ceph-mon[53060]: pgmap v158: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 372 op/s; 427 KiB/s, 10 objects/s recovering 2024-02-07T12:31:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:22.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 248080 10.34 MB/sec execute 429 sec latency 53.084 ms 2024-02-07T12:31:23.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 248377 10.35 MB/sec execute 430 sec latency 38.065 ms 2024-02-07T12:31:23.246 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:23 smithi174 ceph-mon[66222]: pgmap v159: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 538 op/s; 365 KiB/s, 8 objects/s recovering 2024-02-07T12:31:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:23 smithi184 ceph-mon[53060]: pgmap v159: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 538 op/s; 365 KiB/s, 8 objects/s recovering 2024-02-07T12:31:24.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 248713 10.34 MB/sec execute 431 sec latency 57.292 ms 2024-02-07T12:31:24.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:24 smithi174 ceph-mon[66222]: pgmap v160: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 394 op/s 2024-02-07T12:31:25.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 249023 10.34 MB/sec execute 432 sec latency 44.746 ms 2024-02-07T12:31:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:24 smithi184 ceph-mon[53060]: pgmap v160: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 394 op/s 2024-02-07T12:31:25.819 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:31:26.034 INFO:teuthology.orchestra.run.smithi174.stdout: 5 249203 10.33 MB/sec execute 433 sec latency 80.240 ms 2024-02-07T12:31:26.076 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:25 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:31:26.076 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:25 smithi174 ceph-mon[66222]: Upgrade: osd.3 is safe to restart 2024-02-07T12:31:26.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:31:26.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:25 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-02-07T12:31:26.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:25 smithi184 ceph-mon[53060]: Upgrade: osd.3 is safe to restart 2024-02-07T12:31:26.850 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:26 smithi174 ceph-mon[66222]: pgmap v161: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 407 op/s 2024-02-07T12:31:27.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 249488 10.33 MB/sec execute 434 sec latency 59.609 ms 2024-02-07T12:31:27.099 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:26 smithi184 ceph-mon[53060]: pgmap v161: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 407 op/s 2024-02-07T12:31:28.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 249736 10.31 MB/sec execute 435 sec latency 73.375 ms 2024-02-07T12:31:28.913 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:28 smithi174 ceph-mon[66222]: Upgrade: Updating osd.3 2024-02-07T12:31:28.913 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:28 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:28.913 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:28 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:31:28.913 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:28 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:28.913 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:28 smithi174 ceph-mon[66222]: Deploying daemon osd.3 on smithi174 2024-02-07T12:31:28.913 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:28 smithi174 ceph-mon[66222]: pgmap v162: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 571 op/s 2024-02-07T12:31:29.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 250044 10.31 MB/sec execute 436 sec latency 88.304 ms 2024-02-07T12:31:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:28 smithi184 ceph-mon[53060]: Upgrade: Updating osd.3 2024-02-07T12:31:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:28 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:28 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-02-07T12:31:29.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:28 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:28 smithi184 ceph-mon[53060]: Deploying daemon osd.3 on smithi174 2024-02-07T12:31:29.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:28 smithi184 ceph-mon[53060]: pgmap v162: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 571 op/s 2024-02-07T12:31:30.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 250284 10.29 MB/sec execute 437 sec latency 91.783 ms 2024-02-07T12:31:31.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 250477 10.28 MB/sec execute 438 sec latency 48.618 ms 2024-02-07T12:31:31.236 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:31 smithi174 ceph-mon[66222]: pgmap v163: 97 pgs: 97 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-02-07T12:31:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:31 smithi184 ceph-mon[53060]: pgmap v163: 97 pgs: 97 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-02-07T12:31:32.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 250727 10.28 MB/sec execute 439 sec latency 49.503 ms 2024-02-07T12:31:33.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 251106 10.28 MB/sec execute 440 sec latency 59.647 ms 2024-02-07T12:31:33.261 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:33 smithi174 ceph-mon[66222]: pgmap v164: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 564 op/s 2024-02-07T12:31:33.262 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:33 smithi174 systemd[1]: Stopping Ceph osd.3 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:33 smithi184 ceph-mon[53060]: pgmap v164: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 564 op/s 2024-02-07T12:31:33.772 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:33 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[45562]: 2024-02-07T12:31:33.558+0000 7f1d63da8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:31:33.772 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:33 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[45562]: 2024-02-07T12:31:33.558+0000 7f1d63da8700 -1 osd.3 108 *** Got signal Terminated *** 2024-02-07T12:31:33.772 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:33 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[45562]: 2024-02-07T12:31:33.558+0000 7f1d63da8700 -1 osd.3 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:31:34.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 251466 10.27 MB/sec execute 441 sec latency 62.211 ms 2024-02-07T12:31:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:34 smithi184 ceph-mon[53060]: osd.3 marked itself down and dead 2024-02-07T12:31:34.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:34 smithi174 ceph-mon[66222]: osd.3 marked itself down and dead 2024-02-07T12:31:35.003 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:34 smithi174 podman[79861]: 2024-02-07 12:31:34.628057092 +0000 UTC m=+1.200471862 container died 01f07f2e9a5c79d271032dbfc83dc0979e15a8692eaa23d7302ee9a0d3d1c6b7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, GIT_CLEAN=True, io.openshift.expose-services=, version=8, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, release=754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, name=centos-stream) 2024-02-07T12:31:35.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 251499 10.25 MB/sec execute 442 sec latency 902.356 ms 2024-02-07T12:31:35.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: pgmap v165: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 344 op/s 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: osdmap e109: 8 total, 7 up, 8 in 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:35.275 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:35 smithi174 podman[79861]: 2024-02-07 12:31:35.19268393 +0000 UTC m=+1.765098714 container cleanup 01f07f2e9a5c79d271032dbfc83dc0979e15a8692eaa23d7302ee9a0d3d1c6b7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-02-07T12:31:35.276 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:35 smithi174 bash[79861]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3 2024-02-07T12:31:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: pgmap v165: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 344 op/s 2024-02-07T12:31:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:31:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: osdmap e109: 8 total, 7 up, 8 in 2024-02-07T12:31:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: failed to encode map e109 with expected crc 2024-02-07T12:31:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:35.796 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:35 smithi174 podman[79878]: 2024-02-07 12:31:35.552874149 +0000 UTC m=+0.912785903 container remove 01f07f2e9a5c79d271032dbfc83dc0979e15a8692eaa23d7302ee9a0d3d1c6b7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, version=8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, name=centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git) 2024-02-07T12:31:36.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 251753 10.24 MB/sec execute 443 sec latency 1029.458 ms 2024-02-07T12:31:36.052 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:35 smithi174 podman[79964]: 2024-02-07 12:31:35.87445153 +0000 UTC m=+0.020222575 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:31:36.053 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:35 smithi174 podman[79964]: 2024-02-07 12:31:35.995616542 +0000 UTC m=+0.141387561 container create b87e1f55eacaf7352e182b4e0cebfd8da793cca8098a793eacb370a9b68b4931 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, CEPH_POINT_RELEASE=) 2024-02-07T12:31:36.304 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:36 smithi174 podman[79964]: 2024-02-07 12:31:36.290726419 +0000 UTC m=+0.436497439 container init b87e1f55eacaf7352e182b4e0cebfd8da793cca8098a793eacb370a9b68b4931 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True) 2024-02-07T12:31:36.304 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:36 smithi174 podman[79964]: 2024-02-07 12:31:36.297899246 +0000 UTC m=+0.443670269 container start b87e1f55eacaf7352e182b4e0cebfd8da793cca8098a793eacb370a9b68b4931 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_CLEAN=True) 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: osdmap e110: 8 total, 7 up, 8 in 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:36 smithi174 ceph-mon[66222]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: osdmap e110: 8 total, 7 up, 8 in 2024-02-07T12:31:36.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:36 smithi184 ceph-mon[53060]: failed to encode map e110 with expected crc 2024-02-07T12:31:36.559 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:36 smithi174 podman[79964]: 2024-02-07 12:31:36.438966219 +0000 UTC m=+0.584737249 container attach b87e1f55eacaf7352e182b4e0cebfd8da793cca8098a793eacb370a9b68b4931 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:31:36.817 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:36 smithi174 podman[79964]: 2024-02-07 12:31:36.676509936 +0000 UTC m=+0.822280953 container died b87e1f55eacaf7352e182b4e0cebfd8da793cca8098a793eacb370a9b68b4931 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-02-07T12:31:37.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 252047 10.24 MB/sec execute 444 sec latency 41.830 ms 2024-02-07T12:31:37.342 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:37 smithi174 ceph-mon[66222]: pgmap v168: 97 pgs: 9 peering, 13 stale+active+clean, 75 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 MiB/s wr, 260 op/s 2024-02-07T12:31:37.342 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:37 smithi174 ceph-mon[66222]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:31:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:37 smithi184 ceph-mon[53060]: pgmap v168: 97 pgs: 9 peering, 13 stale+active+clean, 75 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 MiB/s wr, 260 op/s 2024-02-07T12:31:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:37 smithi184 ceph-mon[53060]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:31:37.601 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:37 smithi174 podman[79989]: 2024-02-07 12:31:37.561378211 +0000 UTC m=+0.879528890 container remove b87e1f55eacaf7352e182b4e0cebfd8da793cca8098a793eacb370a9b68b4931 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-02-07T12:31:37.601 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:37 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.3.service: Deactivated successfully. 2024-02-07T12:31:37.601 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:37 smithi174 systemd[1]: Stopped Ceph osd.3 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:37.601 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:37 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.3.service: Consumed 5min 17.594s CPU time. 2024-02-07T12:31:38.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 252418 10.24 MB/sec execute 445 sec latency 47.061 ms 2024-02-07T12:31:38.118 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:37 smithi174 systemd[1]: Starting Ceph osd.3 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:31:38.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:38 smithi174 ceph-mon[66222]: Health check failed: Degraded data redundancy: 139/1557 objects degraded (8.927%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:38.376 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:38 smithi174 podman[80093]: 2024-02-07 12:31:38.052670599 +0000 UTC m=+0.024881270 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:31:38.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:38 smithi184 ceph-mon[53060]: Health check failed: Degraded data redundancy: 139/1557 objects degraded (8.927%), 20 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:38.627 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:38 smithi174 podman[80093]: 2024-02-07 12:31:38.379144177 +0000 UTC m=+0.351354831 container create 3a32cca40d15f8c435d3e51607ed111cb41c91d629fb4611ce81d261153dea59 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:31:38.878 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:38 smithi174 podman[80093]: 2024-02-07 12:31:38.746095307 +0000 UTC m=+0.718305978 container init 3a32cca40d15f8c435d3e51607ed111cb41c91d629fb4611ce81d261153dea59 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131) 2024-02-07T12:31:38.878 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:38 smithi174 podman[80093]: 2024-02-07 12:31:38.752739425 +0000 UTC m=+0.724950081 container start 3a32cca40d15f8c435d3e51607ed111cb41c91d629fb4611ce81d261153dea59 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-02-07T12:31:38.878 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:38 smithi174 podman[80093]: 2024-02-07 12:31:38.846154234 +0000 UTC m=+0.818364899 container attach 3a32cca40d15f8c435d3e51607ed111cb41c91d629fb4611ce81d261153dea59 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131) 2024-02-07T12:31:39.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 252724 10.24 MB/sec execute 446 sec latency 71.491 ms 2024-02-07T12:31:39.380 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:39 smithi174 ceph-mon[66222]: pgmap v169: 97 pgs: 12 active+undersized, 20 active+undersized+degraded, 9 peering, 56 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 454 op/s; 139/1557 objects degraded (8.927%) 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate[80108]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate[80108]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 bash[80093]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 bash[80093]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 bash[80093]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 bash[80093]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 bash[80093]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 bash[80093]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 bash[80093]: --> ceph-volume raw activate successful for osd ID: 3 2024-02-07T12:31:39.381 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate[80108]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-02-07T12:31:39.382 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate[80108]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-07T12:31:39.382 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate[80108]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-02-07T12:31:39.382 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate[80108]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-02-07T12:31:39.382 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate[80108]: --> ceph-volume raw activate successful for osd ID: 3 2024-02-07T12:31:39.382 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:39 smithi174 podman[80093]: 2024-02-07 12:31:39.351693816 +0000 UTC m=+1.323904471 container died 3a32cca40d15f8c435d3e51607ed111cb41c91d629fb4611ce81d261153dea59 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, ceph=True, GIT_CLEAN=True) 2024-02-07T12:31:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:39 smithi184 ceph-mon[53060]: pgmap v169: 97 pgs: 12 active+undersized, 20 active+undersized+degraded, 9 peering, 56 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 454 op/s; 139/1557 objects degraded (8.927%) 2024-02-07T12:31:40.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 253031 10.24 MB/sec execute 447 sec latency 52.248 ms 2024-02-07T12:31:40.399 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:40 smithi174 ceph-mon[66222]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-07T12:31:40.652 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:40 smithi174 podman[80239]: 2024-02-07 12:31:40.431896792 +0000 UTC m=+1.066875152 container remove 3a32cca40d15f8c435d3e51607ed111cb41c91d629fb4611ce81d261153dea59 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-activate, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-02-07T12:31:40.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:40 smithi184 ceph-mon[53060]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-02-07T12:31:40.903 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:40 smithi174 podman[80297]: 2024-02-07 12:31:40.617616621 +0000 UTC m=+0.027411619 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:31:40.903 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:40 smithi174 podman[80297]: 2024-02-07 12:31:40.798583932 +0000 UTC m=+0.208378914 container create ea1ed26d44767b80a123e22530a2108d537fa9740027bd00ad5dcd4073bad54d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD) 2024-02-07T12:31:41.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 253332 10.24 MB/sec execute 448 sec latency 94.615 ms 2024-02-07T12:31:41.154 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:41 smithi174 podman[80297]: 2024-02-07 12:31:41.110821337 +0000 UTC m=+0.520616324 container init ea1ed26d44767b80a123e22530a2108d537fa9740027bd00ad5dcd4073bad54d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_BRANCH=HEAD) 2024-02-07T12:31:41.155 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:41 smithi174 podman[80297]: 2024-02-07 12:31:41.121211514 +0000 UTC m=+0.531006492 container start ea1ed26d44767b80a123e22530a2108d537fa9740027bd00ad5dcd4073bad54d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0) 2024-02-07T12:31:41.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:41 smithi174 ceph-mon[66222]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.8 MiB/s wr, 253 op/s; 203/1563 objects degraded (12.988%) 2024-02-07T12:31:41.406 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:41 smithi174 bash[80297]: ea1ed26d44767b80a123e22530a2108d537fa9740027bd00ad5dcd4073bad54d 2024-02-07T12:31:41.406 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:41 smithi174 systemd[1]: Started Ceph osd.3 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:31:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:41 smithi184 ceph-mon[53060]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.8 MiB/s wr, 253 op/s; 203/1563 objects degraded (12.988%) 2024-02-07T12:31:41.908 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:41 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[80311]: 2024-02-07T12:31:41.689+0000 7f55239d57c0 -1 Falling back to public interface 2024-02-07T12:31:42.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 253626 10.22 MB/sec execute 449 sec latency 64.417 ms 2024-02-07T12:31:42.416 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:42 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:42.416 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:42 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:42.417 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:42 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:42 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:42 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:42 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:43.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 253896 10.21 MB/sec execute 450 sec latency 56.948 ms 2024-02-07T12:31:43.472 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:43 smithi174 ceph-mon[66222]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 528 op/s; 207/1626 objects degraded (12.731%) 2024-02-07T12:31:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:43 smithi184 ceph-mon[53060]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 528 op/s; 207/1626 objects degraded (12.731%) 2024-02-07T12:31:44.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 254163 10.20 MB/sec execute 451 sec latency 40.700 ms 2024-02-07T12:31:44.494 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:44 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 207/1626 objects degraded (12.731%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:44 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 207/1626 objects degraded (12.731%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:45.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 254447 10.20 MB/sec execute 452 sec latency 53.602 ms 2024-02-07T12:31:45.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:45 smithi174 ceph-mon[66222]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 425 op/s; 207/1626 objects degraded (12.731%) 2024-02-07T12:31:45.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:45 smithi184 ceph-mon[53060]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 425 op/s; 207/1626 objects degraded (12.731%) 2024-02-07T12:31:46.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 254731 10.20 MB/sec execute 453 sec latency 82.753 ms 2024-02-07T12:31:47.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 255014 10.19 MB/sec execute 454 sec latency 65.175 ms 2024-02-07T12:31:47.255 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:31:47.567 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:47 smithi174 ceph-mon[66222]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.0 MiB/s wr, 389 op/s; 195/1608 objects degraded (12.127%) 2024-02-07T12:31:47.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:47 smithi184 ceph-mon[53060]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.0 MiB/s wr, 389 op/s; 195/1608 objects degraded (12.127%) 2024-02-07T12:31:48.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 255258 10.18 MB/sec execute 455 sec latency 44.581 ms 2024-02-07T12:31:48.299 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (6m) 46s ago 16m 22.8M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (3m) 46s ago 16m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (3m) 3m ago 15m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (6m) 46s ago 15m 51.7M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (10m) 46s ago 10m 38.5M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (10m) 3m ago 10m 13.3M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (5m) 46s ago 17m 506M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (5m) 3m ago 15m 448M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (4m) 46s ago 17m 48.3M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (4m) 3m ago 15m 35.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (9m) 46s ago 9m 253M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (9m) 3m ago 9m 82.5M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:31:48.300 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 46s ago 15m 17.5M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (6m) 3m ago 15m 18.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (2m) 46s ago 14m 310M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (107s) 46s ago 14m 255M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (57s) 46s ago 13m 12.7M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 starting - - - 1936M 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (12m) 3m ago 12m 575M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (12m) 3m ago 12m 442M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (12m) 3m ago 12m 343M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (11m) 3m ago 11m 546M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:31:48.301 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (5m) 46s ago 14m 68.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:31:48.595 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:48 smithi174 ceph-mon[66222]: from='client.34330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:48.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:48 smithi184 ceph-mon[53060]: from='client.34330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:48.906 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 3 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:31:48.907 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:31:48.908 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:31:48.908 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:31:48.908 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-02-07T12:31:48.908 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 7 2024-02-07T12:31:48.908 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:31:48.908 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:31:49.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 255558 10.18 MB/sec execute 456 sec latency 40.097 ms 2024-02-07T12:31:49.369 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:49 smithi174 ceph-mon[66222]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:49.370 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:49 smithi174 ceph-mon[66222]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 511 op/s; 197/1596 objects degraded (12.343%) 2024-02-07T12:31:49.370 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:49 smithi174 ceph-mon[66222]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:49.370 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:49 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 197/1596 objects degraded (12.343%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:49.370 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:49 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3123690732' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "9/23 daemons upgraded", 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:31:49.509 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:31:49.621 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:49 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[80311]: 2024-02-07T12:31:49.448+0000 7f55239d57c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-02-07T12:31:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:49 smithi184 ceph-mon[53060]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:49 smithi184 ceph-mon[53060]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 511 op/s; 197/1596 objects degraded (12.343%) 2024-02-07T12:31:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:49 smithi184 ceph-mon[53060]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:49 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 197/1596 objects degraded (12.343%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:49 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3123690732' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:50.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 255906 10.18 MB/sec execute 457 sec latency 41.867 ms 2024-02-07T12:31:50.223 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 197/1596 objects degraded (12.343%), 27 pgs degraded 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] OSD_DOWN: 1 osds down 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: osd.3 (root=default,host=smithi174) is down 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 197/1596 objects degraded (12.343%), 27 pgs degraded 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-02-07T12:31:50.224 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-02-07T12:31:50.225 INFO:teuthology.orchestra.run.smithi174.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-02-07T12:31:50.226 INFO:teuthology.orchestra.run.smithi174.stdout: pg 4.14 is active+undersized+degraded, acting [2,7] 2024-02-07T12:31:50.226 INFO:teuthology.orchestra.run.smithi174.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-02-07T12:31:50.383 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:50 smithi174 ceph-mon[66222]: from='client.44257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:50.383 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:50.383 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:50.383 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:50.383 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:50.383 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:50 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1879248625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:31:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:50 smithi184 ceph-mon[53060]: from='client.44257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:31:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:31:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:50 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1879248625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:31:51.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 256193 10.17 MB/sec execute 458 sec latency 60.443 ms 2024-02-07T12:31:51.401 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:51 smithi174 ceph-mon[66222]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 386 op/s; 203/1617 objects degraded (12.554%) 2024-02-07T12:31:51.402 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:51 smithi174 ceph-mon[66222]: from='osd.3 [v2:172.21.15.174:6826/1803512753,v1:172.21.15.174:6827/1803512753]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:31:51.407 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:51 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[80311]: 2024-02-07T12:31:51.164+0000 7f55239d57c0 -1 osd.3 108 log_to_monitors true 2024-02-07T12:31:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:51 smithi184 ceph-mon[53060]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 386 op/s; 203/1617 objects degraded (12.554%) 2024-02-07T12:31:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:51 smithi184 ceph-mon[53060]: from='osd.3 [v2:172.21.15.174:6826/1803512753,v1:172.21.15.174:6827/1803512753]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-02-07T12:31:52.035 INFO:teuthology.orchestra.run.smithi174.stdout: 5 256488 10.17 MB/sec execute 459 sec latency 52.590 ms 2024-02-07T12:31:52.169 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:31:51 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[80311]: 2024-02-07T12:31:51.989+0000 7f5514e34700 -1 osd.3 108 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:31:52.689 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:52.689 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:52.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:52 smithi174 ceph-mon[66222]: from='osd.3 [v2:172.21.15.174:6826/1803512753,v1:172.21.15.174:6827/1803512753]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:31:52.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:52 smithi174 ceph-mon[66222]: osdmap e111: 8 total, 7 up, 8 in 2024-02-07T12:31:52.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:52 smithi174 ceph-mon[66222]: from='osd.3 [v2:172.21.15.174:6826/1803512753,v1:172.21.15.174:6827/1803512753]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:31:52.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:52.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:52.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:52 smithi184 ceph-mon[53060]: from='osd.3 [v2:172.21.15.174:6826/1803512753,v1:172.21.15.174:6827/1803512753]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-02-07T12:31:52.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:52 smithi184 ceph-mon[53060]: osdmap e111: 8 total, 7 up, 8 in 2024-02-07T12:31:52.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:52 smithi184 ceph-mon[53060]: from='osd.3 [v2:172.21.15.174:6826/1803512753,v1:172.21.15.174:6827/1803512753]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-02-07T12:31:53.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 256777 10.17 MB/sec execute 460 sec latency 48.767 ms 2024-02-07T12:31:53.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: failed to encode map e111 with expected crc 2024-02-07T12:31:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: failed to encode map e111 with expected crc 2024-02-07T12:31:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 422 op/s; 199/1614 objects degraded (12.330%) 2024-02-07T12:31:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:31:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: osd.3 [v2:172.21.15.174:6826/1803512753,v1:172.21.15.174:6827/1803512753] boot 2024-02-07T12:31:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: osdmap e112: 8 total, 8 up, 8 in 2024-02-07T12:31:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:31:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: failed to encode map e112 with expected crc 2024-02-07T12:31:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:53 smithi174 ceph-mon[66222]: failed to encode map e112 with expected crc 2024-02-07T12:31:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: failed to encode map e111 with expected crc 2024-02-07T12:31:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: failed to encode map e111 with expected crc 2024-02-07T12:31:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 422 op/s; 199/1614 objects degraded (12.330%) 2024-02-07T12:31:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:31:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: osd.3 [v2:172.21.15.174:6826/1803512753,v1:172.21.15.174:6827/1803512753] boot 2024-02-07T12:31:53.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: osdmap e112: 8 total, 8 up, 8 in 2024-02-07T12:31:53.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-02-07T12:31:53.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: failed to encode map e112 with expected crc 2024-02-07T12:31:53.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:53 smithi184 ceph-mon[53060]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 256895 10.15 MB/sec execute 461 sec latency 937.134 ms 2024-02-07T12:31:54.715 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 199/1614 objects degraded (12.330%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: osdmap e113: 8 total, 8 up, 8 in 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.716 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:54 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e112 with expected crc 2024-02-07T12:31:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 199/1614 objects degraded (12.330%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: osdmap e113: 8 total, 8 up, 8 in 2024-02-07T12:31:54.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:54.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:54 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 257030 10.14 MB/sec execute 462 sec latency 1692.091 ms 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.9 MiB/s wr, 357 op/s; 199/1614 objects degraded (12.330%) 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: osdmap e114: 8 total, 8 up, 8 in 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: failed to encode map e114 with expected crc 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: failed to encode map e114 with expected crc 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: failed to encode map e114 with expected crc 2024-02-07T12:31:55.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:55 smithi174 ceph-mon[66222]: failed to encode map e114 with expected crc 2024-02-07T12:31:55.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: failed to encode map e113 with expected crc 2024-02-07T12:31:55.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.9 MiB/s wr, 357 op/s; 199/1614 objects degraded (12.330%) 2024-02-07T12:31:55.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: osdmap e114: 8 total, 8 up, 8 in 2024-02-07T12:31:55.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: failed to encode map e114 with expected crc 2024-02-07T12:31:55.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: failed to encode map e114 with expected crc 2024-02-07T12:31:55.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: failed to encode map e114 with expected crc 2024-02-07T12:31:55.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:55 smithi184 ceph-mon[53060]: failed to encode map e114 with expected crc 2024-02-07T12:31:56.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 257295 10.13 MB/sec execute 463 sec latency 58.656 ms 2024-02-07T12:31:56.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:56 smithi174 ceph-mon[66222]: failed to encode map e114 with expected crc 2024-02-07T12:31:56.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:56 smithi174 ceph-mon[66222]: failed to encode map e114 with expected crc 2024-02-07T12:31:56.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:56 smithi174 ceph-mon[66222]: osdmap e115: 8 total, 8 up, 8 in 2024-02-07T12:31:56.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:56 smithi174 ceph-mon[66222]: failed to encode map e115 with expected crc 2024-02-07T12:31:56.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:56 smithi174 ceph-mon[66222]: failed to encode map e115 with expected crc 2024-02-07T12:31:56.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:56 smithi174 ceph-mon[66222]: failed to encode map e115 with expected crc 2024-02-07T12:31:56.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:56 smithi184 ceph-mon[53060]: failed to encode map e114 with expected crc 2024-02-07T12:31:56.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:56 smithi184 ceph-mon[53060]: failed to encode map e114 with expected crc 2024-02-07T12:31:56.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:56 smithi184 ceph-mon[53060]: osdmap e115: 8 total, 8 up, 8 in 2024-02-07T12:31:56.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:56 smithi184 ceph-mon[53060]: failed to encode map e115 with expected crc 2024-02-07T12:31:56.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:56 smithi184 ceph-mon[53060]: failed to encode map e115 with expected crc 2024-02-07T12:31:56.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:56 smithi184 ceph-mon[53060]: failed to encode map e115 with expected crc 2024-02-07T12:31:57.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 257592 10.12 MB/sec execute 464 sec latency 281.643 ms 2024-02-07T12:31:57.561 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: failed to encode map e115 with expected crc 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: failed to encode map e115 with expected crc 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: failed to encode map e115 with expected crc 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: pgmap v183: 97 pgs: 1 remapped+peering, 12 active+undersized, 23 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 725 KiB/s wr, 30 op/s; 160/1593 objects degraded (10.044%); 42 KiB/s, 7 objects/s recovering 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: osdmap e116: 8 total, 8 up, 8 in 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: failed to encode map e116 with expected crc 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: failed to encode map e116 with expected crc 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: failed to encode map e116 with expected crc 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:57.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:57 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: failed to encode map e115 with expected crc 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: failed to encode map e115 with expected crc 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: failed to encode map e115 with expected crc 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: pgmap v183: 97 pgs: 1 remapped+peering, 12 active+undersized, 23 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 725 KiB/s wr, 30 op/s; 160/1593 objects degraded (10.044%); 42 KiB/s, 7 objects/s recovering 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: osdmap e116: 8 total, 8 up, 8 in 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: failed to encode map e116 with expected crc 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: failed to encode map e116 with expected crc 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: failed to encode map e116 with expected crc 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:57.849 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:31:57.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:31:57.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:57.850 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:57 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:31:58.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 257828 10.11 MB/sec execute 465 sec latency 1029.257 ms 2024-02-07T12:31:58.592 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: failed to encode map e116 with expected crc 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: failed to encode map e116 with expected crc 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: failed to encode map e116 with expected crc 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: osdmap e117: 8 total, 8 up, 8 in 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.594 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.594 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.594 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:58 smithi174 ceph-mon[66222]: pgmap v186: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 remapped+peering, 1 active+undersized+remapped, 91 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 439 op/s; 10/1590 objects degraded (0.629%); 3.2 MiB/s, 51 objects/s recovering 2024-02-07T12:31:58.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: failed to encode map e116 with expected crc 2024-02-07T12:31:58.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: failed to encode map e116 with expected crc 2024-02-07T12:31:58.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: failed to encode map e116 with expected crc 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: osdmap e117: 8 total, 8 up, 8 in 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: failed to encode map e117 with expected crc 2024-02-07T12:31:58.950 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:58 smithi184 ceph-mon[53060]: pgmap v186: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 remapped+peering, 1 active+undersized+remapped, 91 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 439 op/s; 10/1590 objects degraded (0.629%); 3.2 MiB/s, 51 objects/s recovering 2024-02-07T12:31:59.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 258083 10.10 MB/sec execute 466 sec latency 65.653 ms 2024-02-07T12:31:59.585 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:59 smithi174 ceph-mon[66222]: failed to encode map e117 with expected crc 2024-02-07T12:31:59.585 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:31:59 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 160/1593 objects degraded (10.044%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:31:59.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:59 smithi184 ceph-mon[53060]: failed to encode map e117 with expected crc 2024-02-07T12:31:59.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:31:59 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 160/1593 objects degraded (10.044%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:00.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 258344 10.10 MB/sec execute 467 sec latency 62.964 ms 2024-02-07T12:32:00.595 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:00 smithi174 ceph-mon[66222]: pgmap v187: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+undersized+remapped, 92 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.0 MiB/s wr, 330 op/s; 10/1596 objects degraded (0.627%); 2.6 MiB/s, 36 objects/s recovering 2024-02-07T12:32:00.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:00 smithi184 ceph-mon[53060]: pgmap v187: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+undersized+remapped, 92 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.0 MiB/s wr, 330 op/s; 10/1596 objects degraded (0.627%); 2.6 MiB/s, 36 objects/s recovering 2024-02-07T12:32:01.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 258556 10.09 MB/sec execute 468 sec latency 56.247 ms 2024-02-07T12:32:02.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 258887 10.09 MB/sec execute 469 sec latency 52.248 ms 2024-02-07T12:32:02.387 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:02 smithi174 ceph-mon[66222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/1596 objects degraded (0.627%), 1 pg degraded) 2024-02-07T12:32:02.387 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:02 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:32:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:02 smithi184 ceph-mon[53060]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/1596 objects degraded (0.627%), 1 pg degraded) 2024-02-07T12:32:02.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:02 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:32:03.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 259181 10.08 MB/sec execute 470 sec latency 44.121 ms 2024-02-07T12:32:03.423 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:03 smithi174 ceph-mon[66222]: pgmap v188: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.2 MiB/s wr, 566 op/s; 2.2 MiB/s, 32 objects/s recovering 2024-02-07T12:32:03.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:03 smithi184 ceph-mon[53060]: pgmap v188: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.2 MiB/s wr, 566 op/s; 2.2 MiB/s, 32 objects/s recovering 2024-02-07T12:32:04.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 259497 10.08 MB/sec execute 471 sec latency 61.238 ms 2024-02-07T12:32:04.955 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:04 smithi174 ceph-mon[66222]: pgmap v189: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 443 op/s; 1.7 MiB/s, 25 objects/s recovering 2024-02-07T12:32:05.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 259806 10.08 MB/sec execute 472 sec latency 43.611 ms 2024-02-07T12:32:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:04 smithi184 ceph-mon[53060]: pgmap v189: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 443 op/s; 1.7 MiB/s, 25 objects/s recovering 2024-02-07T12:32:06.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 260194 10.08 MB/sec execute 473 sec latency 42.462 ms 2024-02-07T12:32:06.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:06.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:06.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:06.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:07.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 260535 10.07 MB/sec execute 474 sec latency 48.273 ms 2024-02-07T12:32:07.177 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:07 smithi174 ceph-mon[66222]: pgmap v190: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 235 op/s; 157 KiB/s, 12 objects/s recovering 2024-02-07T12:32:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:07 smithi184 ceph-mon[53060]: pgmap v190: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 235 op/s; 157 KiB/s, 12 objects/s recovering 2024-02-07T12:32:08.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 260858 10.07 MB/sec execute 475 sec latency 47.185 ms 2024-02-07T12:32:09.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 261314 10.08 MB/sec execute 476 sec latency 41.646 ms 2024-02-07T12:32:09.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:09 smithi174 ceph-mon[66222]: pgmap v191: 97 pgs: 97 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 357 op/s; 78 KiB/s, 2 objects/s recovering 2024-02-07T12:32:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:09 smithi184 ceph-mon[53060]: pgmap v191: 97 pgs: 97 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 357 op/s; 78 KiB/s, 2 objects/s recovering 2024-02-07T12:32:10.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 261794 10.08 MB/sec execute 477 sec latency 45.976 ms 2024-02-07T12:32:11.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 262102 10.08 MB/sec execute 478 sec latency 52.547 ms 2024-02-07T12:32:11.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:11 smithi174 ceph-mon[66222]: pgmap v192: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 66 KiB/s, 2 objects/s recovering 2024-02-07T12:32:11.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:11 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:11 smithi184 ceph-mon[53060]: pgmap v192: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 66 KiB/s, 2 objects/s recovering 2024-02-07T12:32:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:11 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:12.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 262420 10.07 MB/sec execute 479 sec latency 46.303 ms 2024-02-07T12:32:13.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 262686 10.07 MB/sec execute 480 sec latency 71.995 ms 2024-02-07T12:32:13.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:13 smithi174 ceph-mon[66222]: pgmap v193: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 583 op/s; 0 B/s, 0 objects/s recovering 2024-02-07T12:32:13.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:32:13.567 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:13 smithi184 ceph-mon[53060]: pgmap v193: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 583 op/s; 0 B/s, 0 objects/s recovering 2024-02-07T12:32:13.567 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:32:14.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 262993 10.07 MB/sec execute 481 sec latency 57.696 ms 2024-02-07T12:32:14.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:14 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:32:14.451 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:14 smithi174 ceph-mon[66222]: Upgrade: osd.4 is safe to restart 2024-02-07T12:32:14.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:14 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-02-07T12:32:14.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:14 smithi184 ceph-mon[53060]: Upgrade: osd.4 is safe to restart 2024-02-07T12:32:15.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 263224 10.06 MB/sec execute 482 sec latency 50.206 ms 2024-02-07T12:32:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:15 smithi184 ceph-mon[53060]: pgmap v194: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 433 op/s 2024-02-07T12:32:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:15 smithi184 ceph-mon[53060]: Upgrade: Updating osd.4 2024-02-07T12:32:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:15.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:32:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:15 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:15.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:15 smithi184 ceph-mon[53060]: Deploying daemon osd.4 on smithi184 2024-02-07T12:32:15.724 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:15 smithi174 ceph-mon[66222]: pgmap v194: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 433 op/s 2024-02-07T12:32:15.725 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:15 smithi174 ceph-mon[66222]: Upgrade: Updating osd.4 2024-02-07T12:32:15.725 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:15.725 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-02-07T12:32:15.725 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:15 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:15.725 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:15 smithi174 ceph-mon[66222]: Deploying daemon osd.4 on smithi184 2024-02-07T12:32:16.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 263462 10.05 MB/sec execute 483 sec latency 55.802 ms 2024-02-07T12:32:17.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 263669 10.04 MB/sec execute 484 sec latency 83.824 ms 2024-02-07T12:32:17.535 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:17 smithi174 ceph-mon[66222]: pgmap v195: 97 pgs: 97 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.6 MiB/s wr, 446 op/s 2024-02-07T12:32:17.665 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:17 smithi184 ceph-mon[53060]: pgmap v195: 97 pgs: 97 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.6 MiB/s wr, 446 op/s 2024-02-07T12:32:18.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 263946 10.04 MB/sec execute 485 sec latency 70.796 ms 2024-02-07T12:32:18.374 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:18 smithi184 systemd[1]: Stopping Ceph osd.4 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:32:18.555 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:18 smithi174 ceph-mon[66222]: osd.4 marked itself down and dead 2024-02-07T12:32:18.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:18 smithi184 ceph-mon[53060]: osd.4 marked itself down and dead 2024-02-07T12:32:18.699 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[31245]: 2024-02-07T12:32:18.371+0000 7efd48d68700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:32:18.699 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[31245]: 2024-02-07T12:32:18.371+0000 7efd48d68700 -1 osd.4 117 *** Got signal Terminated *** 2024-02-07T12:32:18.699 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[31245]: 2024-02-07T12:32:18.371+0000 7efd48d68700 -1 osd.4 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:32:19.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 264071 10.02 MB/sec execute 486 sec latency 535.033 ms 2024-02-07T12:32:19.448 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:19 smithi184 podman[57549]: 2024-02-07 12:32:19.023832156 +0000 UTC m=+0.782652519 container died c0d1310d01276024e9541c9554c6d6775c19bed21275f759d68beed0f7a87488 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, architecture=x86_64, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_BRANCH=HEAD, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_CLEAN=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-02-07T12:32:19.735 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:19 smithi174 ceph-mon[66222]: pgmap v196: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 585 op/s 2024-02-07T12:32:19.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:19 smithi174 ceph-mon[66222]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:32:19.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:19 smithi174 ceph-mon[66222]: failed to encode map e118 with expected crc 2024-02-07T12:32:19.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:19 smithi174 ceph-mon[66222]: failed to encode map e118 with expected crc 2024-02-07T12:32:19.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:19 smithi174 ceph-mon[66222]: osdmap e118: 8 total, 7 up, 8 in 2024-02-07T12:32:19.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:19 smithi174 ceph-mon[66222]: failed to encode map e118 with expected crc 2024-02-07T12:32:19.765 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:19 smithi184 ceph-mon[53060]: pgmap v196: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 585 op/s 2024-02-07T12:32:19.766 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:19 smithi184 ceph-mon[53060]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:32:19.766 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:19 smithi184 ceph-mon[53060]: failed to encode map e118 with expected crc 2024-02-07T12:32:19.766 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:19 smithi184 ceph-mon[53060]: failed to encode map e118 with expected crc 2024-02-07T12:32:19.766 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:19 smithi184 ceph-mon[53060]: osdmap e118: 8 total, 7 up, 8 in 2024-02-07T12:32:19.766 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:19 smithi184 ceph-mon[53060]: failed to encode map e118 with expected crc 2024-02-07T12:32:20.037 INFO:teuthology.orchestra.run.smithi174.stdout: 5 264182 10.01 MB/sec execute 487 sec latency 1034.619 ms 2024-02-07T12:32:20.067 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:19 smithi184 podman[57549]: 2024-02-07 12:32:19.764052801 +0000 UTC m=+1.522873166 container cleanup c0d1310d01276024e9541c9554c6d6775c19bed21275f759d68beed0f7a87488 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, release=754, architecture=x86_64, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, name=centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-02-07T12:32:20.067 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:19 smithi184 bash[57549]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4 2024-02-07T12:32:20.435 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:20 smithi184 podman[57566]: 2024-02-07 12:32:20.065778504 +0000 UTC m=+1.046874618 container remove c0d1310d01276024e9541c9554c6d6775c19bed21275f759d68beed0f7a87488 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, vendor=Red Hat, Inc., ceph=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, architecture=x86_64, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: osdmap e119: 8 total, 7 up, 8 in 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:20.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:20.700 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:20 smithi184 podman[57654]: 2024-02-07 12:32:20.336803441 +0000 UTC m=+0.019752265 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:32:20.700 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:20 smithi184 podman[57654]: 2024-02-07 12:32:20.441745627 +0000 UTC m=+0.124694438 container create adfb1a1899912b0f3ec133d8cce66a5325929285ad9d8343df3be956388533af (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-02-07T12:32:20.700 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:20 smithi184 podman[57654]: 2024-02-07 12:32:20.661385045 +0000 UTC m=+0.344333861 container init adfb1a1899912b0f3ec133d8cce66a5325929285ad9d8343df3be956388533af (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:32:20.700 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:20 smithi184 podman[57654]: 2024-02-07 12:32:20.664953639 +0000 UTC m=+0.347902460 container start adfb1a1899912b0f3ec133d8cce66a5325929285ad9d8343df3be956388533af (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-02-07T12:32:20.760 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: osdmap e119: 8 total, 7 up, 8 in 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: failed to encode map e119 with expected crc 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:20.761 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:20.811 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:32:21.036 INFO:teuthology.orchestra.run.smithi174.stdout: 5 264494 10.01 MB/sec execute 488 sec latency 66.997 ms 2024-02-07T12:32:21.385 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:21 smithi184 podman[57654]: 2024-02-07 12:32:21.020285962 +0000 UTC m=+0.703234787 container attach adfb1a1899912b0f3ec133d8cce66a5325929285ad9d8343df3be956388533af (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:32:21.385 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:21 smithi184 podman[57654]: 2024-02-07 12:32:21.020933625 +0000 UTC m=+0.703882439 container died adfb1a1899912b0f3ec133d8cce66a5325929285ad9d8343df3be956388533af (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2) 2024-02-07T12:32:21.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:21 smithi174 ceph-mon[66222]: pgmap v199: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 352 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.9 MiB/s wr, 261 op/s 2024-02-07T12:32:21.659 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:32:21.659 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (7m) 31s ago 16m 22.9M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:32:21.659 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (3m) 31s ago 16m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (3m) 3m ago 15m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (7m) 31s ago 16m 53.2M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (10m) 31s ago 10m 38.7M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (10m) 3m ago 10m 13.3M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (6m) 31s ago 17m 507M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (5m) 3m ago 15m 448M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (5m) 31s ago 18m 50.1M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (4m) 3m ago 15m 35.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (10m) 31s ago 10m 274M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (10m) 3m ago 10m 82.5M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:32:21.660 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 31s ago 16m 17.1M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (6m) 3m ago 15m 18.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (3m) 31s ago 15m 334M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (2m) 31s ago 14m 289M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (90s) 31s ago 14m 235M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (41s) 31s ago 13m 11.8M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (13m) 3m ago 13m 575M 2704M 17.2.0 e1d6a67b021e c0d1310d0127 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (13m) 3m ago 13m 442M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (12m) 3m ago 12m 343M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (12m) 3m ago 12m 546M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:32:21.661 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (5m) 31s ago 15m 68.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:32:21.680 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:21 smithi184 ceph-mon[53060]: pgmap v199: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 352 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.9 MiB/s wr, 261 op/s 2024-02-07T12:32:21.932 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:21 smithi184 podman[57682]: 2024-02-07 12:32:21.675210284 +0000 UTC m=+0.654119433 container remove adfb1a1899912b0f3ec133d8cce66a5325929285ad9d8343df3be956388533af (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0) 2024-02-07T12:32:21.932 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:21 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.4.service: Deactivated successfully. 2024-02-07T12:32:21.932 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:21 smithi184 systemd[1]: Stopped Ceph osd.4 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:32:21.932 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:21 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.4.service: Consumed 5min 11.867s CPU time. 2024-02-07T12:32:22.037 INFO:teuthology.orchestra.run.smithi174.stdout: 5 264763 10.00 MB/sec execute 489 sec latency 39.792 ms 2024-02-07T12:32:22.186 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:21 smithi184 systemd[1]: Starting Ceph osd.4 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:32:22.186 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 podman[57778]: 2024-02-07 12:32:22.071795071 +0000 UTC m=+0.021319722 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 4 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:32:22.187 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:32:22.188 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:32:22.188 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:32:22.188 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:32:22.188 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-02-07T12:32:22.188 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:32:22.188 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:32:22.188 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:32:22.448 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 podman[57778]: 2024-02-07 12:32:22.184959544 +0000 UTC m=+0.134484187 container create 38841370a56758eff5ea695f47fe27b1f646a105bc1a14477a67460689f61348 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:32:22.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:22 smithi174 ceph-mon[66222]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:22.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:22 smithi174 ceph-mon[66222]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:22.545 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:22 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3082118323' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:22.625 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:32:22.625 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:32:22.625 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "10/23 daemons upgraded", 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:32:22.626 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:32:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-mon[53060]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:22.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-mon[53060]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:22.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3082118323' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:22.949 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 podman[57778]: 2024-02-07 12:32:22.454863515 +0000 UTC m=+0.404388158 container init 38841370a56758eff5ea695f47fe27b1f646a105bc1a14477a67460689f61348 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-02-07T12:32:22.949 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 podman[57778]: 2024-02-07 12:32:22.458539751 +0000 UTC m=+0.408064400 container start 38841370a56758eff5ea695f47fe27b1f646a105bc1a14477a67460689f61348 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-02-07T12:32:22.949 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 podman[57778]: 2024-02-07 12:32:22.57190958 +0000 UTC m=+0.521434237 container attach 38841370a56758eff5ea695f47fe27b1f646a105bc1a14477a67460689f61348 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2) 2024-02-07T12:32:23.037 INFO:teuthology.orchestra.run.smithi174.stdout: 5 265034 10.00 MB/sec execute 490 sec latency 62.889 ms 2024-02-07T12:32:23.159 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 203/1596 objects degraded (12.719%), 21 pgs degraded 2024-02-07T12:32:23.159 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] OSD_DOWN: 1 osds down 2024-02-07T12:32:23.159 INFO:teuthology.orchestra.run.smithi174.stdout: osd.4 (root=default,host=smithi184) is down 2024-02-07T12:32:23.159 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 203/1596 objects degraded (12.719%), 21 pgs degraded 2024-02-07T12:32:23.159 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-02-07T12:32:23.159 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-02-07T12:32:23.160 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-02-07T12:32:23.160 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-02-07T12:32:23.160 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-02-07T12:32:23.160 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-02-07T12:32:23.160 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-02-07T12:32:23.160 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-02-07T12:32:23.160 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-02-07T12:32:23.160 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-02-07T12:32:23.161 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-02-07T12:32:23.162 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-02-07T12:32:23.162 INFO:teuthology.orchestra.run.smithi174.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate[57793]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate[57793]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 bash[57778]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 bash[57778]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 bash[57778]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 bash[57778]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 bash[57778]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 bash[57778]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 bash[57778]: --> ceph-volume raw activate successful for osd ID: 4 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate[57793]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate[57793]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-02-07T12:32:23.378 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate[57793]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-02-07T12:32:23.379 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate[57793]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-02-07T12:32:23.379 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:22 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate[57793]: --> ceph-volume raw activate successful for osd ID: 4 2024-02-07T12:32:23.379 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:23 smithi184 podman[57778]: 2024-02-07 12:32:23.01471599 +0000 UTC m=+0.964240633 container died 38841370a56758eff5ea695f47fe27b1f646a105bc1a14477a67460689f61348 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-02-07T12:32:23.565 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:23 smithi174 ceph-mon[66222]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:23.566 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:23 smithi174 ceph-mon[66222]: pgmap v200: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 203/1596 objects degraded (12.719%) 2024-02-07T12:32:23.566 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:23 smithi174 ceph-mon[66222]: Health check failed: Degraded data redundancy: 203/1596 objects degraded (12.719%), 21 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:23.566 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:23 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/2917534889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:32:23.695 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:23 smithi184 ceph-mon[53060]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:23.695 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:23 smithi184 ceph-mon[53060]: pgmap v200: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 203/1596 objects degraded (12.719%) 2024-02-07T12:32:23.695 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:23 smithi184 ceph-mon[53060]: Health check failed: Degraded data redundancy: 203/1596 objects degraded (12.719%), 21 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:23.695 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:23 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/2917534889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:32:23.948 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:23 smithi184 podman[57924]: 2024-02-07 12:32:23.693596153 +0000 UTC m=+0.673137553 container remove 38841370a56758eff5ea695f47fe27b1f646a105bc1a14477a67460689f61348 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:32:23.948 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:23 smithi184 podman[57984]: 2024-02-07 12:32:23.838149007 +0000 UTC m=+0.026765732 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:32:24.037 INFO:teuthology.orchestra.run.smithi174.stdout: 5 265325 9.99 MB/sec execute 491 sec latency 54.838 ms 2024-02-07T12:32:24.448 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:24 smithi184 podman[57984]: 2024-02-07 12:32:24.119946707 +0000 UTC m=+0.308563402 container create 2c6c37d4be2a983b588e0555bcfbbfe252292f0fd408d81141d6fd8549aeec20 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240131) 2024-02-07T12:32:24.577 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:24 smithi174 ceph-mon[66222]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:24.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:24 smithi184 ceph-mon[53060]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:25.037 INFO:teuthology.orchestra.run.smithi174.stdout: 5 265585 9.98 MB/sec execute 492 sec latency 62.885 ms 2024-02-07T12:32:25.358 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:24 smithi184 podman[57984]: 2024-02-07 12:32:24.97471234 +0000 UTC m=+1.163329045 container init 2c6c37d4be2a983b588e0555bcfbbfe252292f0fd408d81141d6fd8549aeec20 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:32:25.359 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:24 smithi184 podman[57984]: 2024-02-07 12:32:24.97817497 +0000 UTC m=+1.166791670 container start 2c6c37d4be2a983b588e0555bcfbbfe252292f0fd408d81141d6fd8549aeec20 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:32:25.359 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:25 smithi184 ceph-osd[58003]: -- 172.21.15.184:0/3395307427 <== mon.1 v2:172.21.15.184:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dfd742d6c0 con 0x55dfd73ff000 2024-02-07T12:32:25.632 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:25 smithi174 ceph-mon[66222]: pgmap v201: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 423 op/s; 203/1596 objects degraded (12.719%) 2024-02-07T12:32:25.633 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:25.633 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:25.633 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:25 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:32:25.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:25 smithi184 ceph-mon[53060]: pgmap v201: 97 pgs: 12 active+undersized, 21 active+undersized+degraded, 8 peering, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 423 op/s; 203/1596 objects degraded (12.719%) 2024-02-07T12:32:25.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:25.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:25.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:25 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:32:25.699 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:25 smithi184 bash[57984]: 2c6c37d4be2a983b588e0555bcfbbfe252292f0fd408d81141d6fd8549aeec20 2024-02-07T12:32:25.699 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:25 smithi184 systemd[1]: Started Ceph osd.4 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:32:25.699 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:25 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:32:25.595+0000 7faff419f7c0 -1 Falling back to public interface 2024-02-07T12:32:26.037 INFO:teuthology.orchestra.run.smithi174.stdout: 5 265826 9.97 MB/sec execute 493 sec latency 67.698 ms 2024-02-07T12:32:27.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 266147 9.98 MB/sec execute 494 sec latency 49.358 ms 2024-02-07T12:32:27.667 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:27 smithi174 ceph-mon[66222]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.3 MiB/s wr, 211 op/s; 257/1599 objects degraded (16.073%) 2024-02-07T12:32:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:27 smithi184 ceph-mon[53060]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.3 MiB/s wr, 211 op/s; 257/1599 objects degraded (16.073%) 2024-02-07T12:32:28.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 266399 9.97 MB/sec execute 495 sec latency 64.766 ms 2024-02-07T12:32:28.687 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:28 smithi174 ceph-mon[66222]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 372 op/s; 245/1566 objects degraded (15.645%) 2024-02-07T12:32:28.688 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:28 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 245/1566 objects degraded (15.645%), 28 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:28.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:28 smithi184 ceph-mon[53060]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 372 op/s; 245/1566 objects degraded (15.645%) 2024-02-07T12:32:28.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:28 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 245/1566 objects degraded (15.645%), 28 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:29.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 266864 9.98 MB/sec execute 496 sec latency 41.009 ms 2024-02-07T12:32:30.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 267181 9.97 MB/sec execute 497 sec latency 58.002 ms 2024-02-07T12:32:30.994 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:30.994 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:30.994 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:30 smithi174 ceph-mon[66222]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 343 op/s; 249/1572 objects degraded (15.840%) 2024-02-07T12:32:31.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 267501 9.97 MB/sec execute 498 sec latency 71.007 ms 2024-02-07T12:32:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:30 smithi184 ceph-mon[53060]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 343 op/s; 249/1572 objects degraded (15.840%) 2024-02-07T12:32:32.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 267752 9.96 MB/sec execute 499 sec latency 41.012 ms 2024-02-07T12:32:32.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:32.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:32.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:32 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:32.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:32 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:32.808 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:32:32.534+0000 7faff419f7c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-02-07T12:32:33.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 268054 9.96 MB/sec execute 500 sec latency 66.893 ms 2024-02-07T12:32:33.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:33 smithi174 ceph-mon[66222]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 494 op/s; 256/1590 objects degraded (16.101%) 2024-02-07T12:32:33.368 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:33 smithi184 ceph-mon[53060]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 494 op/s; 256/1590 objects degraded (16.101%) 2024-02-07T12:32:34.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 268377 9.96 MB/sec execute 501 sec latency 48.970 ms 2024-02-07T12:32:34.198 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:33 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:32:33.883+0000 7faff419f7c0 -1 osd.4 117 log_to_monitors true 2024-02-07T12:32:34.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:34 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 256/1590 objects degraded (16.101%), 28 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:34.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:34 smithi184 ceph-mon[53060]: from='osd.4 [v2:172.21.15.184:6800/3834702541,v1:172.21.15.184:6801/3834702541]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:32:34.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:34 smithi184 ceph-mon[53060]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:32:34.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:34 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 256/1590 objects degraded (16.101%), 28 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:34.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:34 smithi174 ceph-mon[66222]: from='osd.4 [v2:172.21.15.184:6800/3834702541,v1:172.21.15.184:6801/3834702541]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:32:34.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:34 smithi174 ceph-mon[66222]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-02-07T12:32:35.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 268594 9.95 MB/sec execute 502 sec latency 184.018 ms 2024-02-07T12:32:35.448 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:32:35 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:32:35.302+0000 7fafe15f6700 -1 osd.4 117 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:32:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:35 smithi184 ceph-mon[53060]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 373 op/s; 256/1590 objects degraded (16.101%) 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:35 smithi184 ceph-mon[53060]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:35 smithi184 ceph-mon[53060]: osdmap e120: 8 total, 7 up, 8 in 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:35 smithi184 ceph-mon[53060]: from='osd.4 [v2:172.21.15.184:6800/3834702541,v1:172.21.15.184:6801/3834702541]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:35 smithi184 ceph-mon[53060]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:35 smithi174 ceph-mon[66222]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 373 op/s; 256/1590 objects degraded (16.101%) 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:35 smithi174 ceph-mon[66222]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:35 smithi174 ceph-mon[66222]: osdmap e120: 8 total, 7 up, 8 in 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:35 smithi174 ceph-mon[66222]: from='osd.4 [v2:172.21.15.184:6800/3834702541,v1:172.21.15.184:6801/3834702541]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:32:35.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:35 smithi174 ceph-mon[66222]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:32:36.039 INFO:teuthology.orchestra.run.smithi174.stdout: 5 268820 9.94 MB/sec execute 503 sec latency 222.173 ms 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: failed to encode map e120 with expected crc 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: failed to encode map e120 with expected crc 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:32:36.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='osd.4 ' entity='osd.4' 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:32:36.229 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:36 smithi174 ceph-mon[66222]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-02-07T12:32:36.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: failed to encode map e120 with expected crc 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: failed to encode map e120 with expected crc 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='osd.4 ' entity='osd.4' 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:36.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:36.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:36.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:32:36.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:32:36.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:36 smithi184 ceph-mon[53060]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-02-07T12:32:37.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 268958 9.92 MB/sec execute 504 sec latency 722.525 ms 2024-02-07T12:32:37.514 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:37 smithi174 ceph-mon[66222]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 453 op/s; 262/1608 objects degraded (16.294%) 2024-02-07T12:32:37.514 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:37 smithi174 ceph-mon[66222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:32:37.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:37 smithi174 ceph-mon[66222]: failed to encode map e121 with expected crc 2024-02-07T12:32:37.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:37 smithi174 ceph-mon[66222]: osd.4 [v2:172.21.15.184:6800/3834702541,v1:172.21.15.184:6801/3834702541] boot 2024-02-07T12:32:37.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:37 smithi174 ceph-mon[66222]: osdmap e121: 8 total, 8 up, 8 in 2024-02-07T12:32:37.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:37 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:32:37.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:37 smithi184 ceph-mon[53060]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 453 op/s; 262/1608 objects degraded (16.294%) 2024-02-07T12:32:37.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:37 smithi184 ceph-mon[53060]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:32:37.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:37 smithi184 ceph-mon[53060]: failed to encode map e121 with expected crc 2024-02-07T12:32:37.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:37 smithi184 ceph-mon[53060]: osd.4 [v2:172.21.15.184:6800/3834702541,v1:172.21.15.184:6801/3834702541] boot 2024-02-07T12:32:37.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:37 smithi184 ceph-mon[53060]: osdmap e121: 8 total, 8 up, 8 in 2024-02-07T12:32:37.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:37 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-02-07T12:32:38.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 269075 9.91 MB/sec execute 505 sec latency 1722.552 ms 2024-02-07T12:32:38.569 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: osdmap e122: 8 total, 8 up, 8 in 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:38 smithi174 ceph-mon[66222]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: osdmap e122: 8 total, 8 up, 8 in 2024-02-07T12:32:38.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:38.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:38 smithi184 ceph-mon[53060]: failed to encode map e122 with expected crc 2024-02-07T12:32:39.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 269301 9.90 MB/sec execute 506 sec latency 2064.684 ms 2024-02-07T12:32:39.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: pgmap v211: 97 pgs: 2 peering, 12 active+undersized, 27 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.0 MiB/s wr, 333 op/s; 228/1590 objects degraded (14.340%) 2024-02-07T12:32:39.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: osdmap e123: 8 total, 8 up, 8 in 2024-02-07T12:32:39.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.336 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:39 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 228/1590 objects degraded (14.340%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:39.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: pgmap v211: 97 pgs: 2 peering, 12 active+undersized, 27 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.0 MiB/s wr, 333 op/s; 228/1590 objects degraded (14.340%) 2024-02-07T12:32:39.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: osdmap e123: 8 total, 8 up, 8 in 2024-02-07T12:32:39.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: failed to encode map e123 with expected crc 2024-02-07T12:32:39.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:39 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 228/1590 objects degraded (14.340%), 27 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:40.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 269604 9.90 MB/sec execute 507 sec latency 63.276 ms 2024-02-07T12:32:41.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 269867 9.90 MB/sec execute 508 sec latency 335.925 ms 2024-02-07T12:32:41.592 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:41 smithi174 ceph-mon[66222]: pgmap v213: 97 pgs: 1 active+recovery_wait+degraded, 2 peering, 2 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized+degraded, 73 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.1 MiB/s wr, 363 op/s; 177/1548 objects degraded (11.434%); 49 KiB/s, 11 objects/s recovering 2024-02-07T12:32:41.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:41 smithi174 ceph-mon[66222]: failed to encode map e124 with expected crc 2024-02-07T12:32:41.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:41 smithi174 ceph-mon[66222]: osdmap e124: 8 total, 8 up, 8 in 2024-02-07T12:32:41.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:41 smithi174 ceph-mon[66222]: failed to encode map e124 with expected crc 2024-02-07T12:32:41.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:41 smithi174 ceph-mon[66222]: failed to encode map e124 with expected crc 2024-02-07T12:32:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:41 smithi184 ceph-mon[53060]: pgmap v213: 97 pgs: 1 active+recovery_wait+degraded, 2 peering, 2 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized+degraded, 73 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.1 MiB/s wr, 363 op/s; 177/1548 objects degraded (11.434%); 49 KiB/s, 11 objects/s recovering 2024-02-07T12:32:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:41 smithi184 ceph-mon[53060]: failed to encode map e124 with expected crc 2024-02-07T12:32:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:41 smithi184 ceph-mon[53060]: osdmap e124: 8 total, 8 up, 8 in 2024-02-07T12:32:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:41 smithi184 ceph-mon[53060]: failed to encode map e124 with expected crc 2024-02-07T12:32:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:41 smithi184 ceph-mon[53060]: failed to encode map e124 with expected crc 2024-02-07T12:32:42.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 270177 9.90 MB/sec execute 509 sec latency 694.177 ms 2024-02-07T12:32:42.373 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:42 smithi174 ceph-mon[66222]: failed to encode map e125 with expected crc 2024-02-07T12:32:42.374 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:42 smithi174 ceph-mon[66222]: failed to encode map e125 with expected crc 2024-02-07T12:32:42.374 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:42 smithi174 ceph-mon[66222]: osdmap e125: 8 total, 8 up, 8 in 2024-02-07T12:32:42.374 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:42 smithi174 ceph-mon[66222]: failed to encode map e125 with expected crc 2024-02-07T12:32:42.374 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:42 smithi174 ceph-mon[66222]: failed to encode map e125 with expected crc 2024-02-07T12:32:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:42 smithi184 ceph-mon[53060]: failed to encode map e125 with expected crc 2024-02-07T12:32:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:42 smithi184 ceph-mon[53060]: failed to encode map e125 with expected crc 2024-02-07T12:32:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:42 smithi184 ceph-mon[53060]: osdmap e125: 8 total, 8 up, 8 in 2024-02-07T12:32:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:42 smithi184 ceph-mon[53060]: failed to encode map e125 with expected crc 2024-02-07T12:32:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:42 smithi184 ceph-mon[53060]: failed to encode map e125 with expected crc 2024-02-07T12:32:43.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 270475 9.89 MB/sec execute 510 sec latency 1031.688 ms 2024-02-07T12:32:43.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:43 smithi174 ceph-mon[66222]: pgmap v216: 97 pgs: 1 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 90 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 343 op/s; 94/1554 objects degraded (6.049%); 100 KiB/s, 53 keys/s, 39 objects/s recovering 2024-02-07T12:32:43.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:43 smithi174 ceph-mon[66222]: failed to encode map e126 with expected crc 2024-02-07T12:32:43.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:43 smithi174 ceph-mon[66222]: osdmap e126: 8 total, 8 up, 8 in 2024-02-07T12:32:43.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:43 smithi174 ceph-mon[66222]: failed to encode map e126 with expected crc 2024-02-07T12:32:43.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:43 smithi174 ceph-mon[66222]: failed to encode map e126 with expected crc 2024-02-07T12:32:43.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:43 smithi174 ceph-mon[66222]: failed to encode map e126 with expected crc 2024-02-07T12:32:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:43 smithi184 ceph-mon[53060]: pgmap v216: 97 pgs: 1 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 90 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 343 op/s; 94/1554 objects degraded (6.049%); 100 KiB/s, 53 keys/s, 39 objects/s recovering 2024-02-07T12:32:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:43 smithi184 ceph-mon[53060]: failed to encode map e126 with expected crc 2024-02-07T12:32:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:43 smithi184 ceph-mon[53060]: osdmap e126: 8 total, 8 up, 8 in 2024-02-07T12:32:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:43 smithi184 ceph-mon[53060]: failed to encode map e126 with expected crc 2024-02-07T12:32:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:43 smithi184 ceph-mon[53060]: failed to encode map e126 with expected crc 2024-02-07T12:32:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:43 smithi184 ceph-mon[53060]: failed to encode map e126 with expected crc 2024-02-07T12:32:44.041 INFO:teuthology.orchestra.run.smithi174.stdout: 5 270713 9.89 MB/sec execute 511 sec latency 342.617 ms 2024-02-07T12:32:44.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:44 smithi174 ceph-mon[66222]: failed to encode map e127 with expected crc 2024-02-07T12:32:44.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:44 smithi174 ceph-mon[66222]: osdmap e127: 8 total, 8 up, 8 in 2024-02-07T12:32:44.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:44 smithi174 ceph-mon[66222]: failed to encode map e127 with expected crc 2024-02-07T12:32:44.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:44 smithi174 ceph-mon[66222]: failed to encode map e127 with expected crc 2024-02-07T12:32:44.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:44 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 94/1554 objects degraded (6.049%), 6 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:44.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:44 smithi174 ceph-mon[66222]: osdmap e128: 8 total, 8 up, 8 in 2024-02-07T12:32:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:44 smithi184 ceph-mon[53060]: failed to encode map e127 with expected crc 2024-02-07T12:32:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:44 smithi184 ceph-mon[53060]: osdmap e127: 8 total, 8 up, 8 in 2024-02-07T12:32:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:44 smithi184 ceph-mon[53060]: failed to encode map e127 with expected crc 2024-02-07T12:32:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:44 smithi184 ceph-mon[53060]: failed to encode map e127 with expected crc 2024-02-07T12:32:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:44 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 94/1554 objects degraded (6.049%), 6 pgs degraded (PG_DEGRADED) 2024-02-07T12:32:44.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:44 smithi184 ceph-mon[53060]: osdmap e128: 8 total, 8 up, 8 in 2024-02-07T12:32:45.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 271021 9.88 MB/sec execute 512 sec latency 850.416 ms 2024-02-07T12:32:45.480 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:45 smithi174 ceph-mon[66222]: failed to encode map e128 with expected crc 2024-02-07T12:32:45.480 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:45 smithi174 ceph-mon[66222]: failed to encode map e128 with expected crc 2024-02-07T12:32:45.480 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:45 smithi174 ceph-mon[66222]: failed to encode map e128 with expected crc 2024-02-07T12:32:45.480 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:45 smithi174 ceph-mon[66222]: pgmap v220: 97 pgs: 1 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 90 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.4 MiB/s wr, 408 op/s; 94/1554 objects degraded (6.049%); 58 KiB/s, 67 keys/s, 34 objects/s recovering 2024-02-07T12:32:45.480 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:45 smithi174 ceph-mon[66222]: osdmap e129: 8 total, 8 up, 8 in 2024-02-07T12:32:45.481 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:45 smithi174 ceph-mon[66222]: failed to encode map e129 with expected crc 2024-02-07T12:32:45.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:45 smithi184 ceph-mon[53060]: failed to encode map e128 with expected crc 2024-02-07T12:32:45.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:45 smithi184 ceph-mon[53060]: failed to encode map e128 with expected crc 2024-02-07T12:32:45.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:45 smithi184 ceph-mon[53060]: failed to encode map e128 with expected crc 2024-02-07T12:32:45.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:45 smithi184 ceph-mon[53060]: pgmap v220: 97 pgs: 1 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 90 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.4 MiB/s wr, 408 op/s; 94/1554 objects degraded (6.049%); 58 KiB/s, 67 keys/s, 34 objects/s recovering 2024-02-07T12:32:45.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:45 smithi184 ceph-mon[53060]: osdmap e129: 8 total, 8 up, 8 in 2024-02-07T12:32:45.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:45 smithi184 ceph-mon[53060]: failed to encode map e129 with expected crc 2024-02-07T12:32:46.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 271282 9.88 MB/sec execute 513 sec latency 62.895 ms 2024-02-07T12:32:46.496 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:46 smithi174 ceph-mon[66222]: failed to encode map e129 with expected crc 2024-02-07T12:32:46.496 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:46 smithi174 ceph-mon[66222]: failed to encode map e129 with expected crc 2024-02-07T12:32:46.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:46 smithi184 ceph-mon[53060]: failed to encode map e129 with expected crc 2024-02-07T12:32:46.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:46 smithi184 ceph-mon[53060]: failed to encode map e129 with expected crc 2024-02-07T12:32:47.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 271622 9.88 MB/sec execute 514 sec latency 50.627 ms 2024-02-07T12:32:47.508 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:47 smithi174 ceph-mon[66222]: pgmap v222: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 59 op/s; 45/1578 objects degraded (2.852%); 577 KiB/s, 14 objects/s recovering 2024-02-07T12:32:47.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:47 smithi184 ceph-mon[53060]: pgmap v222: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 59 op/s; 45/1578 objects degraded (2.852%); 577 KiB/s, 14 objects/s recovering 2024-02-07T12:32:48.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 271986 9.88 MB/sec execute 515 sec latency 70.064 ms 2024-02-07T12:32:48.526 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:48 smithi174 ceph-mon[66222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1578 objects degraded (2.852%), 3 pgs degraded) 2024-02-07T12:32:48.527 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:48 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:32:48.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:48 smithi184 ceph-mon[53060]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1578 objects degraded (2.852%), 3 pgs degraded) 2024-02-07T12:32:48.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:48 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:32:49.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 272317 9.87 MB/sec execute 516 sec latency 47.859 ms 2024-02-07T12:32:49.573 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:49 smithi174 ceph-mon[66222]: pgmap v223: 97 pgs: 97 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 359 op/s; 402 KiB/s, 20 objects/s recovering 2024-02-07T12:32:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:49 smithi184 ceph-mon[53060]: pgmap v223: 97 pgs: 97 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 359 op/s; 402 KiB/s, 20 objects/s recovering 2024-02-07T12:32:50.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 272639 9.87 MB/sec execute 517 sec latency 54.070 ms 2024-02-07T12:32:51.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 272935 9.86 MB/sec execute 518 sec latency 45.147 ms 2024-02-07T12:32:51.331 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:51 smithi184 ceph-mon[53060]: pgmap v224: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.3 MiB/s wr, 346 op/s; 343 KiB/s, 17 objects/s recovering 2024-02-07T12:32:51.332 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:51.332 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:51.332 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:32:51.332 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:51 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:32:51.332 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:51 smithi184 ceph-mon[53060]: Upgrade: osd.5 is safe to restart 2024-02-07T12:32:51.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:51 smithi174 ceph-mon[66222]: pgmap v224: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.3 MiB/s wr, 346 op/s; 343 KiB/s, 17 objects/s recovering 2024-02-07T12:32:51.368 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:51.369 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:32:51.369 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:32:51.369 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:51 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-02-07T12:32:51.369 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:51 smithi174 ceph-mon[66222]: Upgrade: osd.5 is safe to restart 2024-02-07T12:32:52.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 273247 9.86 MB/sec execute 519 sec latency 42.348 ms 2024-02-07T12:32:53.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 273563 9.86 MB/sec execute 520 sec latency 47.669 ms 2024-02-07T12:32:53.640 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:32:53.645 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:53 smithi174 ceph-mon[66222]: pgmap v225: 97 pgs: 97 active+clean; 359 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 571 op/s; 278 KiB/s, 14 objects/s recovering 2024-02-07T12:32:53.645 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:53 smithi174 ceph-mon[66222]: Upgrade: Updating osd.5 2024-02-07T12:32:53.645 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:53.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:32:53.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:53.646 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:53 smithi174 ceph-mon[66222]: Deploying daemon osd.5 on smithi184 2024-02-07T12:32:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:53 smithi184 ceph-mon[53060]: pgmap v225: 97 pgs: 97 active+clean; 359 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 571 op/s; 278 KiB/s, 14 objects/s recovering 2024-02-07T12:32:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:53 smithi184 ceph-mon[53060]: Upgrade: Updating osd.5 2024-02-07T12:32:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-02-07T12:32:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:32:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:53 smithi184 ceph-mon[53060]: Deploying daemon osd.5 on smithi184 2024-02-07T12:32:54.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 273877 9.86 MB/sec execute 521 sec latency 55.338 ms 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (8m) 64s ago 17m 22.9M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (4m) 64s ago 17m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (4m) 24s ago 16m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (7m) 64s ago 16m 53.2M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (11m) 64s ago 11m 38.7M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (11m) 24s ago 11m 14.0M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (6m) 64s ago 18m 507M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (6m) 24s ago 16m 450M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (5m) 64s ago 18m 50.1M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (5m) 24s ago 16m 42.4M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (10m) 64s ago 10m 274M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:32:54.592 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (10m) 24s ago 10m 87.0M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 64s ago 16m 17.1M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (7m) 24s ago 16m 17.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (3m) 64s ago 15m 334M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (2m) 64s ago 15m 289M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (2m) 64s ago 14m 235M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (73s) 64s ago 14m 11.8M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (30s) 24s ago 13m 15.2M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (13m) 24s ago 13m 639M 2704M 17.2.0 e1d6a67b021e 55fc87e8637e 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (13m) 24s ago 13m 549M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (12m) 24s ago 12m 583M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:32:54.593 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (6m) 64s ago 15m 68.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:32:54.918 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:54 smithi174 ceph-mon[66222]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:54.919 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:54 smithi174 ceph-mon[66222]: pgmap v226: 97 pgs: 97 active+clean; 359 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 474 op/s; 231 KiB/s, 11 objects/s recovering 2024-02-07T12:32:54.919 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:54 smithi174 ceph-mon[66222]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:55.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 274186 9.86 MB/sec execute 522 sec latency 51.241 ms 2024-02-07T12:32:55.069 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:54 smithi184 ceph-mon[53060]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:55.069 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:54 smithi184 ceph-mon[53060]: pgmap v226: 97 pgs: 97 active+clean; 359 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 474 op/s; 231 KiB/s, 11 objects/s recovering 2024-02-07T12:32:55.069 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:54 smithi184 ceph-mon[53060]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:55.120 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:32:55.120 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:32:55.120 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 5 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 9 2024-02-07T12:32:55.121 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:32:55.122 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "11/23 daemons upgraded", 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:32:55.552 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:32:55.553 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:32:55.946 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:55 smithi174 ceph-mon[66222]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:55.946 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:55 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/131486288' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:55.946 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:55 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:55.946 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:55 smithi174 ceph-mon[66222]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:56.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 274389 9.85 MB/sec execute 523 sec latency 47.016 ms 2024-02-07T12:32:56.078 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:32:56.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:55 smithi184 ceph-mon[53060]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:56.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:55 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/131486288' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:32:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:55 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:32:56.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:55 smithi184 ceph-mon[53060]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:32:56.532 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:56 smithi184 systemd[1]: Stopping Ceph osd.5 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:32:56.830 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:56 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[34229]: 2024-02-07T12:32:56.529+0000 7f36d1cc3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:32:56.830 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:56 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[34229]: 2024-02-07T12:32:56.529+0000 7f36d1cc3700 -1 osd.5 129 *** Got signal Terminated *** 2024-02-07T12:32:56.830 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:56 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[34229]: 2024-02-07T12:32:56.529+0000 7f36d1cc3700 -1 osd.5 129 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:32:57.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 274634 9.84 MB/sec execute 524 sec latency 225.994 ms 2024-02-07T12:32:57.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:56 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1298774751' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:32:57.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:56 smithi174 ceph-mon[66222]: pgmap v227: 97 pgs: 97 active+clean; 359 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.5 MiB/s wr, 430 op/s; 133 KiB/s, 7 objects/s recovering 2024-02-07T12:32:57.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:56 smithi174 ceph-mon[66222]: osd.5 marked itself down and dead 2024-02-07T12:32:57.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:56 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1298774751' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:32:57.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:56 smithi184 ceph-mon[53060]: pgmap v227: 97 pgs: 97 active+clean; 359 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.5 MiB/s wr, 430 op/s; 133 KiB/s, 7 objects/s recovering 2024-02-07T12:32:57.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:56 smithi184 ceph-mon[53060]: osd.5 marked itself down and dead 2024-02-07T12:32:57.790 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:57 smithi184 podman[59804]: 2024-02-07 12:32:57.488359448 +0000 UTC m=+1.096214811 container died 55fc87e8637e08de1203f260a739a9187945be5596d82b512bd381bfb01f4d09 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , RELEASE=HEAD, ceph=True, release=754, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, architecture=x86_64, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-02-07T12:32:58.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 274657 9.82 MB/sec execute 525 sec latency 1053.227 ms 2024-02-07T12:32:58.079 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:57 smithi174 ceph-mon[66222]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:32:58.079 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:57 smithi174 ceph-mon[66222]: failed to encode map e130 with expected crc 2024-02-07T12:32:58.080 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:57 smithi174 ceph-mon[66222]: osdmap e130: 8 total, 7 up, 8 in 2024-02-07T12:32:58.080 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:57 smithi174 ceph-mon[66222]: failed to encode map e130 with expected crc 2024-02-07T12:32:58.080 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:57 smithi174 ceph-mon[66222]: failed to encode map e130 with expected crc 2024-02-07T12:32:58.080 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:57 smithi174 ceph-mon[66222]: failed to encode map e130 with expected crc 2024-02-07T12:32:58.158 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:57 smithi184 ceph-mon[53060]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:32:58.158 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:57 smithi184 ceph-mon[53060]: failed to encode map e130 with expected crc 2024-02-07T12:32:58.158 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:57 smithi184 ceph-mon[53060]: osdmap e130: 8 total, 7 up, 8 in 2024-02-07T12:32:58.158 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:57 smithi184 ceph-mon[53060]: failed to encode map e130 with expected crc 2024-02-07T12:32:58.159 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:57 smithi184 ceph-mon[53060]: failed to encode map e130 with expected crc 2024-02-07T12:32:58.159 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:57 smithi184 ceph-mon[53060]: failed to encode map e130 with expected crc 2024-02-07T12:32:58.159 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:57 smithi184 podman[59804]: 2024-02-07 12:32:57.879633583 +0000 UTC m=+1.487488946 container cleanup 55fc87e8637e08de1203f260a739a9187945be5596d82b512bd381bfb01f4d09 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_CLEAN=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream) 2024-02-07T12:32:58.159 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:57 smithi184 bash[59804]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5 2024-02-07T12:32:58.448 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:58 smithi184 podman[59822]: 2024-02-07 12:32:58.156218018 +0000 UTC m=+0.667074890 container remove 55fc87e8637e08de1203f260a739a9187945be5596d82b512bd381bfb01f4d09 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, io.buildah.version=1.19.8, architecture=x86_64, vcs-type=git, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.component=centos-stream-container, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD) 2024-02-07T12:32:58.770 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:58 smithi184 podman[59909]: 2024-02-07 12:32:58.400336269 +0000 UTC m=+0.020487945 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:32:58.770 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:58 smithi184 podman[59909]: 2024-02-07 12:32:58.515643787 +0000 UTC m=+0.135795457 container create 936074ce90ebad85da5dc20a0b8abc4ecaf1d514849679778ae22a826212593d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=) 2024-02-07T12:32:58.770 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:58 smithi184 podman[59909]: 2024-02-07 12:32:58.768575758 +0000 UTC m=+0.388727427 container init 936074ce90ebad85da5dc20a0b8abc4ecaf1d514849679778ae22a826212593d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:32:59.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 274943 9.82 MB/sec execute 526 sec latency 49.834 ms 2024-02-07T12:32:59.115 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:58 smithi174 ceph-mon[66222]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.116 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:58 smithi174 ceph-mon[66222]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.116 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:58 smithi174 ceph-mon[66222]: osdmap e131: 8 total, 7 up, 8 in 2024-02-07T12:32:59.116 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:58 smithi174 ceph-mon[66222]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.116 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:58 smithi174 ceph-mon[66222]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.116 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:58 smithi174 ceph-mon[66222]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.116 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:58 smithi174 ceph-mon[66222]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.116 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:58 smithi174 ceph-mon[66222]: pgmap v230: 97 pgs: 5 peering, 9 stale+active+clean, 83 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.8 MiB/s wr, 545 op/s 2024-02-07T12:32:59.125 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:58 smithi184 podman[59909]: 2024-02-07 12:32:58.771979672 +0000 UTC m=+0.392131349 container start 936074ce90ebad85da5dc20a0b8abc4ecaf1d514849679778ae22a826212593d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:32:59.125 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:58 smithi184 podman[59909]: 2024-02-07 12:32:58.86884078 +0000 UTC m=+0.488992462 container attach 936074ce90ebad85da5dc20a0b8abc4ecaf1d514849679778ae22a826212593d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:32:59.125 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:58 smithi184 ceph-mon[53060]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.125 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:58 smithi184 ceph-mon[53060]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.125 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:58 smithi184 ceph-mon[53060]: osdmap e131: 8 total, 7 up, 8 in 2024-02-07T12:32:59.125 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:58 smithi184 ceph-mon[53060]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.125 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:58 smithi184 ceph-mon[53060]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.125 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:58 smithi184 ceph-mon[53060]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.125 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:58 smithi184 ceph-mon[53060]: failed to encode map e131 with expected crc 2024-02-07T12:32:59.126 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:58 smithi184 ceph-mon[53060]: pgmap v230: 97 pgs: 5 peering, 9 stale+active+clean, 83 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.8 MiB/s wr, 545 op/s 2024-02-07T12:32:59.432 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:59 smithi184 podman[59909]: 2024-02-07 12:32:59.123315433 +0000 UTC m=+0.743467103 container died 936074ce90ebad85da5dc20a0b8abc4ecaf1d514849679778ae22a826212593d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:32:59.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:32:59 smithi174 ceph-mon[66222]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:32:59.985 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:32:59 smithi184 ceph-mon[53060]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:32:59.985 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:59 smithi184 podman[59935]: 2024-02-07 12:32:59.715051419 +0000 UTC m=+0.583126914 container remove 936074ce90ebad85da5dc20a0b8abc4ecaf1d514849679778ae22a826212593d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-02-07T12:32:59.985 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:59 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.5.service: Deactivated successfully. 2024-02-07T12:32:59.985 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:59 smithi184 systemd[1]: Stopped Ceph osd.5 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:32:59.985 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:59 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.5.service: Consumed 5min 51.956s CPU time. 2024-02-07T12:33:00.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 275250 9.82 MB/sec execute 527 sec latency 57.989 ms 2024-02-07T12:33:00.243 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:32:59 smithi184 systemd[1]: Starting Ceph osd.5 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:33:00.243 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:00 smithi184 podman[60034]: 2024-02-07 12:33:00.127850683 +0000 UTC m=+0.024587177 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:33:00.243 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:00 smithi184 podman[60034]: 2024-02-07 12:33:00.24149976 +0000 UTC m=+0.138236242 container create b80b0f6f633ef1efbe437124fbe017e3c0e5014986cdfaacdbb53c4b7906c768 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2) 2024-02-07T12:33:00.869 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:00 smithi184 podman[60034]: 2024-02-07 12:33:00.52806901 +0000 UTC m=+0.424805494 container init b80b0f6f633ef1efbe437124fbe017e3c0e5014986cdfaacdbb53c4b7906c768 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240131, ceph=True) 2024-02-07T12:33:00.869 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:00 smithi184 podman[60034]: 2024-02-07 12:33:00.53207858 +0000 UTC m=+0.428815092 container start b80b0f6f633ef1efbe437124fbe017e3c0e5014986cdfaacdbb53c4b7906c768 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:33:00.869 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:00 smithi184 podman[60034]: 2024-02-07 12:33:00.628101179 +0000 UTC m=+0.524837661 container attach b80b0f6f633ef1efbe437124fbe017e3c0e5014986cdfaacdbb53c4b7906c768 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-02-07T12:33:00.905 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:00 smithi174 ceph-mon[66222]: pgmap v231: 97 pgs: 3 active+undersized, 5 active+undersized+degraded, 5 peering, 6 stale+active+clean, 78 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.0 MiB/s wr, 269 op/s; 46/1599 objects degraded (2.877%) 2024-02-07T12:33:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 275604 9.82 MB/sec execute 528 sec latency 46.804 ms 2024-02-07T12:33:01.119 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:00 smithi184 ceph-mon[53060]: pgmap v231: 97 pgs: 3 active+undersized, 5 active+undersized+degraded, 5 peering, 6 stale+active+clean, 78 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.0 MiB/s wr, 269 op/s; 46/1599 objects degraded (2.877%) 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate[60049]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate[60049]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 bash[60034]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 bash[60034]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 bash[60034]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 bash[60034]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 bash[60034]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 bash[60034]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 bash[60034]: --> ceph-volume raw activate successful for osd ID: 5 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate[60049]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate[60049]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate[60049]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-02-07T12:33:01.120 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate[60049]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-02-07T12:33:01.121 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate[60049]: --> ceph-volume raw activate successful for osd ID: 5 2024-02-07T12:33:01.121 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 podman[60034]: 2024-02-07 12:33:01.118098067 +0000 UTC m=+1.014834548 container died b80b0f6f633ef1efbe437124fbe017e3c0e5014986cdfaacdbb53c4b7906c768 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate, io.buildah.version=1.33.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-02-07T12:33:01.918 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:01 smithi174 ceph-mon[66222]: Health check failed: Degraded data redundancy: 46/1599 objects degraded (2.877%), 5 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:02.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 275971 9.81 MB/sec execute 529 sec latency 46.547 ms 2024-02-07T12:33:02.123 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:01 smithi184 ceph-mon[53060]: Health check failed: Degraded data redundancy: 46/1599 objects degraded (2.877%), 5 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:02.124 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:01 smithi184 podman[60181]: 2024-02-07 12:33:01.883041418 +0000 UTC m=+0.759311601 container remove b80b0f6f633ef1efbe437124fbe017e3c0e5014986cdfaacdbb53c4b7906c768 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-activate, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=) 2024-02-07T12:33:02.393 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:02 smithi184 podman[60241]: 2024-02-07 12:33:02.0250806 +0000 UTC m=+0.022837474 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:33:02.394 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:02 smithi184 podman[60241]: 2024-02-07 12:33:02.152995341 +0000 UTC m=+0.150752220 container create 8b5693e2bfd0d830a564626ec9dc97e6f69c37de05acd323a7cc0336353eb4eb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-02-07T12:33:02.887 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:02 smithi184 podman[60241]: 2024-02-07 12:33:02.520595586 +0000 UTC m=+0.518352462 container init 8b5693e2bfd0d830a564626ec9dc97e6f69c37de05acd323a7cc0336353eb4eb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-02-07T12:33:02.888 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:02 smithi184 podman[60241]: 2024-02-07 12:33:02.524581499 +0000 UTC m=+0.522338380 container start 8b5693e2bfd0d830a564626ec9dc97e6f69c37de05acd323a7cc0336353eb4eb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, io.buildah.version=1.33.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:33:02.888 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:02 smithi184 ceph-osd[60259]: -- 172.21.15.184:0/2539845064 <== mon.1 v2:172.21.15.184:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c5433cf6c0 con 0x55c5433a1000 2024-02-07T12:33:02.888 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:02 smithi184 bash[60241]: 8b5693e2bfd0d830a564626ec9dc97e6f69c37de05acd323a7cc0336353eb4eb 2024-02-07T12:33:02.888 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:02 smithi184 systemd[1]: Started Ceph osd.5 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:33:02.931 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:02 smithi174 ceph-mon[66222]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 420 op/s; 164/1596 objects degraded (10.276%) 2024-02-07T12:33:02.932 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:02.932 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:02.932 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:02 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:03.041 INFO:teuthology.orchestra.run.smithi174.stdout: 5 276316 9.81 MB/sec execute 530 sec latency 61.872 ms 2024-02-07T12:33:03.198 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:03 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:33:03.070+0000 7f9dafb317c0 -1 Falling back to public interface 2024-02-07T12:33:03.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:02 smithi184 ceph-mon[53060]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 420 op/s; 164/1596 objects degraded (10.276%) 2024-02-07T12:33:03.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:03.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:03.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:02 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:03.935 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:03 smithi184 ceph-mon[53060]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-02-07T12:33:03.957 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:03 smithi174 ceph-mon[66222]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-02-07T12:33:04.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 276641 9.81 MB/sec execute 531 sec latency 43.051 ms 2024-02-07T12:33:04.972 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:04 smithi174 ceph-mon[66222]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 390 op/s; 164/1596 objects degraded (10.276%) 2024-02-07T12:33:05.041 INFO:teuthology.orchestra.run.smithi174.stdout: 5 276894 9.80 MB/sec execute 532 sec latency 47.168 ms 2024-02-07T12:33:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:04 smithi184 ceph-mon[53060]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 390 op/s; 164/1596 objects degraded (10.276%) 2024-02-07T12:33:06.040 INFO:teuthology.orchestra.run.smithi174.stdout: 5 277120 9.80 MB/sec execute 533 sec latency 73.889 ms 2024-02-07T12:33:06.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:06.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:06.422 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:06.423 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:07.041 INFO:teuthology.orchestra.run.smithi174.stdout: 5 277433 9.80 MB/sec execute 534 sec latency 48.768 ms 2024-02-07T12:33:07.334 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:07 smithi174 ceph-mon[66222]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.7 MiB/s wr, 408 op/s; 163/1608 objects degraded (10.137%) 2024-02-07T12:33:07.334 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:07 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 163/1608 objects degraded (10.137%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:07.334 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:07 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:07.334 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:07 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:07 smithi184 ceph-mon[53060]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.7 MiB/s wr, 408 op/s; 163/1608 objects degraded (10.137%) 2024-02-07T12:33:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:07 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 163/1608 objects degraded (10.137%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:07 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:07 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:08.041 INFO:teuthology.orchestra.run.smithi174.stdout: 5 277772 9.80 MB/sec execute 535 sec latency 51.533 ms 2024-02-07T12:33:09.048 INFO:teuthology.orchestra.run.smithi174.stdout: 5 278085 9.79 MB/sec execute 536 sec latency 40.450 ms 2024-02-07T12:33:09.380 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:09 smithi174 ceph-mon[66222]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 357 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 424 op/s; 168/1599 objects degraded (10.507%) 2024-02-07T12:33:09.380 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:09.381 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:09 smithi184 ceph-mon[53060]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 357 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 424 op/s; 168/1599 objects degraded (10.507%) 2024-02-07T12:33:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:10.055 INFO:teuthology.orchestra.run.smithi174.stdout: 5 278367 9.79 MB/sec execute 537 sec latency 48.000 ms 2024-02-07T12:33:11.055 INFO:teuthology.orchestra.run.smithi174.stdout: 5 278698 9.79 MB/sec execute 538 sec latency 50.095 ms 2024-02-07T12:33:11.211 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:11 smithi174 ceph-mon[66222]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 357 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 383 op/s; 168/1602 objects degraded (10.487%) 2024-02-07T12:33:11.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:11 smithi184 ceph-mon[53060]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 357 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 383 op/s; 168/1602 objects degraded (10.487%) 2024-02-07T12:33:12.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 279041 9.79 MB/sec execute 539 sec latency 44.293 ms 2024-02-07T12:33:12.198 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:11 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:33:11.768+0000 7f9dafb317c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-02-07T12:33:13.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 279345 9.78 MB/sec execute 540 sec latency 47.079 ms 2024-02-07T12:33:13.242 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 358 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 539 op/s; 172/1620 objects degraded (10.617%) 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:33:13.243 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:12 smithi174 ceph-mon[66222]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-02-07T12:33:13.448 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:13 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:33:13.175+0000 7f9dafb317c0 -1 osd.5 129 log_to_monitors true 2024-02-07T12:33:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:13.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 358 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 539 op/s; 172/1620 objects degraded (10.617%) 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:33:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:12 smithi184 ceph-mon[53060]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-02-07T12:33:14.015 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:13 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 172/1620 objects degraded (10.617%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:14.015 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:13 smithi174 ceph-mon[66222]: from='osd.5 [v2:172.21.15.184:6808/854642974,v1:172.21.15.184:6809/854642974]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:33:14.015 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:13 smithi174 ceph-mon[66222]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:33:14.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 279708 9.78 MB/sec execute 541 sec latency 55.011 ms 2024-02-07T12:33:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:13 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 172/1620 objects degraded (10.617%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:13 smithi184 ceph-mon[53060]: from='osd.5 [v2:172.21.15.184:6808/854642974,v1:172.21.15.184:6809/854642974]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:33:14.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:13 smithi184 ceph-mon[53060]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-02-07T12:33:14.449 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:33:14 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:33:14.040+0000 7f9da0f90700 -1 osd.5 129 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:33:15.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:15 smithi174 ceph-mon[66222]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:33:15.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:15 smithi174 ceph-mon[66222]: failed to encode map e132 with expected crc 2024-02-07T12:33:15.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:15 smithi174 ceph-mon[66222]: osdmap e132: 8 total, 7 up, 8 in 2024-02-07T12:33:15.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:15 smithi174 ceph-mon[66222]: from='osd.5 [v2:172.21.15.184:6808/854642974,v1:172.21.15.184:6809/854642974]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:33:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:15 smithi174 ceph-mon[66222]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:33:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:15 smithi174 ceph-mon[66222]: failed to encode map e132 with expected crc 2024-02-07T12:33:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:15 smithi174 ceph-mon[66222]: failed to encode map e132 with expected crc 2024-02-07T12:33:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:15 smithi174 ceph-mon[66222]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 358 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 526 op/s; 172/1620 objects degraded (10.617%) 2024-02-07T12:33:15.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 279988 9.77 MB/sec execute 542 sec latency 46.618 ms 2024-02-07T12:33:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:15 smithi184 ceph-mon[53060]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-02-07T12:33:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:15 smithi184 ceph-mon[53060]: failed to encode map e132 with expected crc 2024-02-07T12:33:15.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:15 smithi184 ceph-mon[53060]: osdmap e132: 8 total, 7 up, 8 in 2024-02-07T12:33:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:15 smithi184 ceph-mon[53060]: from='osd.5 [v2:172.21.15.184:6808/854642974,v1:172.21.15.184:6809/854642974]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:33:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:15 smithi184 ceph-mon[53060]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:33:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:15 smithi184 ceph-mon[53060]: failed to encode map e132 with expected crc 2024-02-07T12:33:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:15 smithi184 ceph-mon[53060]: failed to encode map e132 with expected crc 2024-02-07T12:33:15.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:15 smithi184 ceph-mon[53060]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 358 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 526 op/s; 172/1620 objects degraded (10.617%) 2024-02-07T12:33:16.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 280063 9.76 MB/sec execute 543 sec latency 1002.567 ms 2024-02-07T12:33:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:33:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: osd.5 [v2:172.21.15.184:6808/854642974,v1:172.21.15.184:6809/854642974] boot 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: osdmap e133: 8 total, 8 up, 8 in 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:16 smithi184 ceph-mon[53060]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:33:16.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: osd.5 [v2:172.21.15.184:6808/854642974,v1:172.21.15.184:6809/854642974] boot 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: osdmap e133: 8 total, 8 up, 8 in 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: failed to encode map e133 with expected crc 2024-02-07T12:33:16.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:16 smithi174 ceph-mon[66222]: failed to encode map e133 with expected crc 2024-02-07T12:33:17.059 INFO:teuthology.orchestra.run.smithi174.stdout: 5 280066 9.74 MB/sec execute 544 sec latency 2002.595 ms 2024-02-07T12:33:17.350 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: osdmap e134: 8 total, 8 up, 8 in 2024-02-07T12:33:17.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.351 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 360 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 398 op/s; 170/1626 objects degraded (10.455%) 2024-02-07T12:33:17.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:17 smithi174 ceph-mon[66222]: osdmap e135: 8 total, 8 up, 8 in 2024-02-07T12:33:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: osdmap e134: 8 total, 8 up, 8 in 2024-02-07T12:33:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: failed to encode map e134 with expected crc 2024-02-07T12:33:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 360 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 398 op/s; 170/1626 objects degraded (10.455%) 2024-02-07T12:33:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:17 smithi184 ceph-mon[53060]: osdmap e135: 8 total, 8 up, 8 in 2024-02-07T12:33:18.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 280285 9.74 MB/sec execute 545 sec latency 2120.719 ms 2024-02-07T12:33:18.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:18 smithi174 ceph-mon[66222]: failed to encode map e135 with expected crc 2024-02-07T12:33:18.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:18 smithi174 ceph-mon[66222]: failed to encode map e135 with expected crc 2024-02-07T12:33:18.363 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:18 smithi174 ceph-mon[66222]: failed to encode map e135 with expected crc 2024-02-07T12:33:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:18 smithi184 ceph-mon[53060]: failed to encode map e135 with expected crc 2024-02-07T12:33:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:18 smithi184 ceph-mon[53060]: failed to encode map e135 with expected crc 2024-02-07T12:33:18.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:18 smithi184 ceph-mon[53060]: failed to encode map e135 with expected crc 2024-02-07T12:33:19.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 280632 9.75 MB/sec execute 546 sec latency 245.698 ms 2024-02-07T12:33:19.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:19 smithi174 ceph-mon[66222]: pgmap v244: 97 pgs: 1 active+recovering+degraded, 5 remapped+peering, 5 active+undersized, 11 active+undersized+degraded, 1 active+degraded, 74 active+clean; 360 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 407 op/s; 95/1644 objects degraded (5.779%); 22 B/s, 0 objects/s recovering 2024-02-07T12:33:19.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:19 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 95/1644 objects degraded (5.779%), 13 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:19.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:19 smithi174 ceph-mon[66222]: failed to encode map e136 with expected crc 2024-02-07T12:33:19.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:19 smithi174 ceph-mon[66222]: osdmap e136: 8 total, 8 up, 8 in 2024-02-07T12:33:19.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:19 smithi174 ceph-mon[66222]: failed to encode map e136 with expected crc 2024-02-07T12:33:19.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:19 smithi174 ceph-mon[66222]: failed to encode map e136 with expected crc 2024-02-07T12:33:19.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:19 smithi174 ceph-mon[66222]: osdmap e137: 8 total, 8 up, 8 in 2024-02-07T12:33:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:19 smithi184 ceph-mon[53060]: pgmap v244: 97 pgs: 1 active+recovering+degraded, 5 remapped+peering, 5 active+undersized, 11 active+undersized+degraded, 1 active+degraded, 74 active+clean; 360 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 407 op/s; 95/1644 objects degraded (5.779%); 22 B/s, 0 objects/s recovering 2024-02-07T12:33:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:19 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 95/1644 objects degraded (5.779%), 13 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:19 smithi184 ceph-mon[53060]: failed to encode map e136 with expected crc 2024-02-07T12:33:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:19 smithi184 ceph-mon[53060]: osdmap e136: 8 total, 8 up, 8 in 2024-02-07T12:33:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:19 smithi184 ceph-mon[53060]: failed to encode map e136 with expected crc 2024-02-07T12:33:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:19 smithi184 ceph-mon[53060]: failed to encode map e136 with expected crc 2024-02-07T12:33:19.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:19 smithi184 ceph-mon[53060]: osdmap e137: 8 total, 8 up, 8 in 2024-02-07T12:33:20.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 280885 9.74 MB/sec execute 547 sec latency 1029.690 ms 2024-02-07T12:33:20.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:20 smithi174 ceph-mon[66222]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:20 smithi174 ceph-mon[66222]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:20 smithi174 ceph-mon[66222]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:20 smithi174 ceph-mon[66222]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:20 smithi174 ceph-mon[66222]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:20 smithi174 ceph-mon[66222]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:20 smithi174 ceph-mon[66222]: osdmap e138: 8 total, 8 up, 8 in 2024-02-07T12:33:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:20 smithi184 ceph-mon[53060]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:20 smithi184 ceph-mon[53060]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:20 smithi184 ceph-mon[53060]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:20 smithi184 ceph-mon[53060]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:20 smithi184 ceph-mon[53060]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:20 smithi184 ceph-mon[53060]: failed to encode map e137 with expected crc 2024-02-07T12:33:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:20 smithi184 ceph-mon[53060]: osdmap e138: 8 total, 8 up, 8 in 2024-02-07T12:33:21.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 281183 9.74 MB/sec execute 548 sec latency 59.662 ms 2024-02-07T12:33:21.209 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:21 smithi174 ceph-mon[66222]: failed to encode map e138 with expected crc 2024-02-07T12:33:21.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:21 smithi174 ceph-mon[66222]: failed to encode map e138 with expected crc 2024-02-07T12:33:21.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:21 smithi174 ceph-mon[66222]: failed to encode map e138 with expected crc 2024-02-07T12:33:21.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:21 smithi174 ceph-mon[66222]: pgmap v248: 97 pgs: 1 peering, 1 active+recovering+degraded, 4 remapped+peering, 1 active+degraded, 90 active+clean; 359 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 392 op/s; 7/1611 objects degraded (0.435%); 270 KiB/s, 28 objects/s recovering 2024-02-07T12:33:21.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:21.210 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:21 smithi184 ceph-mon[53060]: failed to encode map e138 with expected crc 2024-02-07T12:33:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:21 smithi184 ceph-mon[53060]: failed to encode map e138 with expected crc 2024-02-07T12:33:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:21 smithi184 ceph-mon[53060]: failed to encode map e138 with expected crc 2024-02-07T12:33:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:21 smithi184 ceph-mon[53060]: pgmap v248: 97 pgs: 1 peering, 1 active+recovering+degraded, 4 remapped+peering, 1 active+degraded, 90 active+clean; 359 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 392 op/s; 7/1611 objects degraded (0.435%); 270 KiB/s, 28 objects/s recovering 2024-02-07T12:33:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:22.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 281477 9.73 MB/sec execute 549 sec latency 44.674 ms 2024-02-07T12:33:22.244 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:22 smithi174 ceph-mon[66222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/1611 objects degraded (0.435%), 2 pgs degraded) 2024-02-07T12:33:22.244 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:22 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:33:22.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:22 smithi184 ceph-mon[53060]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/1611 objects degraded (0.435%), 2 pgs degraded) 2024-02-07T12:33:22.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:22 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:33:23.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 281727 9.72 MB/sec execute 550 sec latency 51.781 ms 2024-02-07T12:33:23.280 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:23 smithi174 ceph-mon[66222]: pgmap v249: 97 pgs: 1 peering, 96 active+clean; 363 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.7 MiB/s wr, 355 op/s; 871 KiB/s, 39 objects/s recovering 2024-02-07T12:33:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:23 smithi184 ceph-mon[53060]: pgmap v249: 97 pgs: 1 peering, 96 active+clean; 363 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.7 MiB/s wr, 355 op/s; 871 KiB/s, 39 objects/s recovering 2024-02-07T12:33:24.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 282031 9.71 MB/sec execute 551 sec latency 68.139 ms 2024-02-07T12:33:25.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 282399 9.72 MB/sec execute 552 sec latency 42.066 ms 2024-02-07T12:33:25.058 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:24 smithi174 ceph-mon[66222]: pgmap v250: 97 pgs: 97 active+clean; 359 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 309 op/s; 738 KiB/s, 37 objects/s recovering 2024-02-07T12:33:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:24 smithi184 ceph-mon[53060]: pgmap v250: 97 pgs: 97 active+clean; 359 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 309 op/s; 738 KiB/s, 37 objects/s recovering 2024-02-07T12:33:26.057 INFO:teuthology.orchestra.run.smithi174.stdout: 5 282745 9.72 MB/sec execute 553 sec latency 48.731 ms 2024-02-07T12:33:26.610 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:33:27.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 283116 9.72 MB/sec execute 554 sec latency 42.894 ms 2024-02-07T12:33:27.331 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:27 smithi174 ceph-mon[66222]: pgmap v251: 97 pgs: 97 active+clean; 358 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.1 MiB/s wr, 286 op/s; 554 KiB/s, 28 objects/s recovering 2024-02-07T12:33:27.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:27 smithi184 ceph-mon[53060]: pgmap v251: 97 pgs: 97 active+clean; 358 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.1 MiB/s wr, 286 op/s; 554 KiB/s, 28 objects/s recovering 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (8m) 97s ago 17m 22.9M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (4m) 97s ago 17m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (4m) 20s ago 17m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (8m) 97s ago 17m 53.2M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (11m) 97s ago 11m 38.7M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (11m) 20s ago 11m 14.2M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (7m) 97s ago 18m 507M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:33:27.558 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (6m) 20s ago 16m 450M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (6m) 97s ago 19m 50.1M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (5m) 20s ago 16m 41.6M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (11m) 97s ago 11m 274M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (11m) 20s ago 11m 87.1M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 97s ago 17m 17.1M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (7m) 20s ago 16m 21.0M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (4m) 97s ago 16m 334M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (3m) 97s ago 15m 289M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (2m) 97s ago 15m 235M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (106s) 97s ago 14m 11.8M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (63s) 20s ago 14m 277M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (25s) 20s ago 14m 14.8M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (13m) 20s ago 13m 584M 2704M 17.2.0 e1d6a67b021e 7857992596a7 2024-02-07T12:33:27.559 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (13m) 20s ago 13m 624M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:33:27.560 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (6m) 97s ago 16m 68.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:33:28.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 283387 9.71 MB/sec execute 555 sec latency 79.988 ms 2024-02-07T12:33:28.078 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 6 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:33:28.079 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-02-07T12:33:28.080 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 10 2024-02-07T12:33:28.080 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:33:28.080 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:33:28.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:28 smithi184 ceph-mon[53060]: from='client.44299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:33:28.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:28 smithi184 ceph-mon[53060]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:33:28.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:28 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:33:28.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:28 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:33:28.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:28 smithi184 ceph-mon[53060]: Upgrade: osd.6 is safe to restart 2024-02-07T12:33:28.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:28 smithi184 ceph-mon[53060]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:33:28.290 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:28 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/240166764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:28.381 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:28 smithi174 ceph-mon[66222]: from='client.44299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:33:28.381 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:28 smithi174 ceph-mon[66222]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:33:28.381 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:28 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:33:28.381 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:28 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-02-07T12:33:28.382 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:28 smithi174 ceph-mon[66222]: Upgrade: osd.6 is safe to restart 2024-02-07T12:33:28.382 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:28 smithi174 ceph-mon[66222]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:33:28.382 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:28 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/240166764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "12/23 daemons upgraded", 2024-02-07T12:33:28.512 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:33:28.513 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:33:28.513 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:33:29.040 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:33:29.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 283675 9.71 MB/sec execute 556 sec latency 70.683 ms 2024-02-07T12:33:29.220 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:29 smithi174 ceph-mon[66222]: pgmap v252: 97 pgs: 97 active+clean; 361 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 373 op/s; 476 KiB/s, 24 objects/s recovering 2024-02-07T12:33:29.220 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:29 smithi174 ceph-mon[66222]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:33:29.221 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:29 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1872811485' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:33:29.221 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:29.221 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:33:29.221 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:29 smithi184 ceph-mon[53060]: pgmap v252: 97 pgs: 97 active+clean; 361 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 373 op/s; 476 KiB/s, 24 objects/s recovering 2024-02-07T12:33:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:29 smithi184 ceph-mon[53060]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:33:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:29 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1872811485' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:33:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-02-07T12:33:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:30.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 283891 9.70 MB/sec execute 557 sec latency 60.962 ms 2024-02-07T12:33:30.253 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:30 smithi174 ceph-mon[66222]: Upgrade: Updating osd.6 2024-02-07T12:33:30.253 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:30 smithi174 ceph-mon[66222]: Deploying daemon osd.6 on smithi184 2024-02-07T12:33:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:30 smithi184 ceph-mon[53060]: Upgrade: Updating osd.6 2024-02-07T12:33:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:30 smithi184 ceph-mon[53060]: Deploying daemon osd.6 on smithi184 2024-02-07T12:33:31.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 284222 9.71 MB/sec execute 558 sec latency 56.270 ms 2024-02-07T12:33:31.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:31 smithi174 ceph-mon[66222]: pgmap v253: 97 pgs: 97 active+clean; 365 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.8 MiB/s wr, 526 op/s; 325 KiB/s, 10 objects/s recovering 2024-02-07T12:33:31.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:31.357 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:31 smithi184 ceph-mon[53060]: pgmap v253: 97 pgs: 97 active+clean; 365 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.8 MiB/s wr, 526 op/s; 325 KiB/s, 10 objects/s recovering 2024-02-07T12:33:31.357 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:32.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 284539 9.70 MB/sec execute 559 sec latency 67.628 ms 2024-02-07T12:33:32.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:32 smithi184 systemd[1]: Stopping Ceph osd.6 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:33:33.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 284861 9.70 MB/sec execute 560 sec latency 49.034 ms 2024-02-07T12:33:33.240 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[37211]: 2024-02-07T12:33:32.982+0000 7f991026a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:33:33.241 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[37211]: 2024-02-07T12:33:32.982+0000 7f991026a700 -1 osd.6 138 *** Got signal Terminated *** 2024-02-07T12:33:33.241 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[37211]: 2024-02-07T12:33:32.982+0000 7f991026a700 -1 osd.6 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:33:33.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:33 smithi174 ceph-mon[66222]: pgmap v254: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 487 op/s; 279 KiB/s, 9 objects/s recovering 2024-02-07T12:33:33.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:33 smithi174 ceph-mon[66222]: osd.6 marked itself down and dead 2024-02-07T12:33:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:33 smithi184 ceph-mon[53060]: pgmap v254: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 487 op/s; 279 KiB/s, 9 objects/s recovering 2024-02-07T12:33:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:33 smithi184 ceph-mon[53060]: osd.6 marked itself down and dead 2024-02-07T12:33:34.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 285049 9.69 MB/sec execute 561 sec latency 767.376 ms 2024-02-07T12:33:34.195 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:33 smithi184 podman[62023]: 2024-02-07 12:33:33.857110659 +0000 UTC m=+1.001627638 container died 7857992596a77d4afc1e27351776ec07dba1a942e3a82bc4a70f0a30a04f4142 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, ceph=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, distribution-scope=public, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container) 2024-02-07T12:33:34.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:34 smithi174 ceph-mon[66222]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:33:34.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:34 smithi174 ceph-mon[66222]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:34 smithi174 ceph-mon[66222]: osdmap e139: 8 total, 7 up, 8 in 2024-02-07T12:33:34.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:34 smithi174 ceph-mon[66222]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:34 smithi174 ceph-mon[66222]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:34 smithi174 ceph-mon[66222]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.360 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:34 smithi174 ceph-mon[66222]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.448 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:34 smithi184 podman[62023]: 2024-02-07 12:33:34.291500454 +0000 UTC m=+1.436017445 container cleanup 7857992596a77d4afc1e27351776ec07dba1a942e3a82bc4a70f0a30a04f4142 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, release=754, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, version=8, architecture=x86_64, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-02-07T12:33:34.448 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:34 smithi184 bash[62023]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6 2024-02-07T12:33:34.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:34 smithi184 ceph-mon[53060]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:33:34.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:34 smithi184 ceph-mon[53060]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:34 smithi184 ceph-mon[53060]: osdmap e139: 8 total, 7 up, 8 in 2024-02-07T12:33:34.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:34 smithi184 ceph-mon[53060]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:34 smithi184 ceph-mon[53060]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:34 smithi184 ceph-mon[53060]: failed to encode map e139 with expected crc 2024-02-07T12:33:34.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:34 smithi184 ceph-mon[53060]: failed to encode map e139 with expected crc 2024-02-07T12:33:35.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 285293 9.68 MB/sec execute 562 sec latency 1006.663 ms 2024-02-07T12:33:35.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:34 smithi184 podman[62072]: 2024-02-07 12:33:34.760147015 +0000 UTC m=+0.905989277 container remove 7857992596a77d4afc1e27351776ec07dba1a942e3a82bc4a70f0a30a04f4142 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, version=8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, distribution-scope=public, GIT_CLEAN=True, architecture=x86_64, RELEASE=HEAD, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vcs-type=git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, release=754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870) 2024-02-07T12:33:35.400 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:35 smithi174 ceph-mon[66222]: pgmap v256: 97 pgs: 7 peering, 8 stale+active+clean, 82 active+clean; 361 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.5 MiB/s wr, 435 op/s 2024-02-07T12:33:35.400 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:35 smithi174 ceph-mon[66222]: failed to encode map e140 with expected crc 2024-02-07T12:33:35.400 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:35 smithi174 ceph-mon[66222]: osdmap e140: 8 total, 7 up, 8 in 2024-02-07T12:33:35.400 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:35 smithi174 ceph-mon[66222]: failed to encode map e140 with expected crc 2024-02-07T12:33:35.400 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:35.400 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:35.516 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:35 smithi184 podman[62159]: 2024-02-07 12:33:35.115373693 +0000 UTC m=+0.020745783 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:33:35.517 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:35 smithi184 podman[62159]: 2024-02-07 12:33:35.227740461 +0000 UTC m=+0.133112522 container create a006f9675fbfeca69f936b65d6a91f76a11a7525cfaf617b495eeb1f55ec3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.2) 2024-02-07T12:33:35.517 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:35 smithi184 ceph-mon[53060]: pgmap v256: 97 pgs: 7 peering, 8 stale+active+clean, 82 active+clean; 361 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.5 MiB/s wr, 435 op/s 2024-02-07T12:33:35.517 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:35 smithi184 ceph-mon[53060]: failed to encode map e140 with expected crc 2024-02-07T12:33:35.517 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:35 smithi184 ceph-mon[53060]: osdmap e140: 8 total, 7 up, 8 in 2024-02-07T12:33:35.517 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:35 smithi184 ceph-mon[53060]: failed to encode map e140 with expected crc 2024-02-07T12:33:35.517 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:35.517 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:35.858 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:35 smithi184 podman[62159]: 2024-02-07 12:33:35.514806866 +0000 UTC m=+0.420178940 container init a006f9675fbfeca69f936b65d6a91f76a11a7525cfaf617b495eeb1f55ec3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131) 2024-02-07T12:33:35.858 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:35 smithi184 podman[62159]: 2024-02-07 12:33:35.519303173 +0000 UTC m=+0.424675243 container start a006f9675fbfeca69f936b65d6a91f76a11a7525cfaf617b495eeb1f55ec3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-02-07T12:33:35.858 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:35 smithi184 podman[62159]: 2024-02-07 12:33:35.623136207 +0000 UTC m=+0.528508281 container attach a006f9675fbfeca69f936b65d6a91f76a11a7525cfaf617b495eeb1f55ec3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:33:35.858 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:35 smithi184 podman[62159]: 2024-02-07 12:33:35.855749628 +0000 UTC m=+0.761121697 container died a006f9675fbfeca69f936b65d6a91f76a11a7525cfaf617b495eeb1f55ec3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:33:36.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 285529 9.67 MB/sec execute 563 sec latency 51.631 ms 2024-02-07T12:33:36.418 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:36 smithi174 ceph-mon[66222]: Health check failed: Degraded data redundancy: 40/1605 objects degraded (2.492%), 6 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:36.621 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:36 smithi184 ceph-mon[53060]: Health check failed: Degraded data redundancy: 40/1605 objects degraded (2.492%), 6 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:36.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:36 smithi184 podman[62185]: 2024-02-07 12:33:36.620081681 +0000 UTC m=+0.758835823 container remove a006f9675fbfeca69f936b65d6a91f76a11a7525cfaf617b495eeb1f55ec3176 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-02-07T12:33:36.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:36 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.6.service: Deactivated successfully. 2024-02-07T12:33:36.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:36 smithi184 systemd[1]: Stopped Ceph osd.6 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:33:36.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:36 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.6.service: Consumed 5min 20.220s CPU time. 2024-02-07T12:33:36.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:36 smithi184 systemd[1]: Starting Ceph osd.6 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:33:37.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 285857 9.67 MB/sec execute 564 sec latency 56.641 ms 2024-02-07T12:33:37.392 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:37 smithi184 ceph-mon[53060]: pgmap v258: 97 pgs: 5 active+undersized, 6 active+undersized+degraded, 8 peering, 3 stale+active+clean, 75 active+clean; 362 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 40/1605 objects degraded (2.492%) 2024-02-07T12:33:37.393 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 podman[62287]: 2024-02-07 12:33:36.996583448 +0000 UTC m=+0.018656673 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:33:37.393 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 podman[62287]: 2024-02-07 12:33:37.112774365 +0000 UTC m=+0.134847590 container create 317f7f35f6d0c74c830db89cbedc5d9e8f928772adf9115d25cbebf6a4c11f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240131, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:33:37.393 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 podman[62287]: 2024-02-07 12:33:37.390847213 +0000 UTC m=+0.412920450 container init 317f7f35f6d0c74c830db89cbedc5d9e8f928772adf9115d25cbebf6a4c11f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux ) 2024-02-07T12:33:37.461 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:37 smithi174 ceph-mon[66222]: pgmap v258: 97 pgs: 5 active+undersized, 6 active+undersized+degraded, 8 peering, 3 stale+active+clean, 75 active+clean; 362 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 40/1605 objects degraded (2.492%) 2024-02-07T12:33:37.698 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 podman[62287]: 2024-02-07 12:33:37.394807325 +0000 UTC m=+0.416880552 container start 317f7f35f6d0c74c830db89cbedc5d9e8f928772adf9115d25cbebf6a4c11f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate, io.buildah.version=1.33.2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:33:37.698 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 podman[62287]: 2024-02-07 12:33:37.499425883 +0000 UTC m=+0.521499118 container attach 317f7f35f6d0c74c830db89cbedc5d9e8f928772adf9115d25cbebf6a4c11f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:33:38.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 286180 9.67 MB/sec execute 565 sec latency 78.545 ms 2024-02-07T12:33:38.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate[62302]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate[62302]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 bash[62287]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 bash[62287]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 bash[62287]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 bash[62287]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 bash[62287]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 bash[62287]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 bash[62287]: --> ceph-volume raw activate successful for osd ID: 6 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate[62302]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate[62302]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate[62302]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate[62302]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-02-07T12:33:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate[62302]: --> ceph-volume raw activate successful for osd ID: 6 2024-02-07T12:33:38.200 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:37 smithi184 podman[62287]: 2024-02-07 12:33:37.924528582 +0000 UTC m=+0.946601810 container died 317f7f35f6d0c74c830db89cbedc5d9e8f928772adf9115d25cbebf6a4c11f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True) 2024-02-07T12:33:39.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 286540 9.67 MB/sec execute 566 sec latency 47.194 ms 2024-02-07T12:33:39.143 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:38 smithi184 podman[62434]: 2024-02-07 12:33:38.704842808 +0000 UTC m=+0.775259374 container remove 317f7f35f6d0c74c830db89cbedc5d9e8f928772adf9115d25cbebf6a4c11f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-activate, GIT_CLEAN=True, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240131) 2024-02-07T12:33:39.143 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:38 smithi184 podman[62496]: 2024-02-07 12:33:38.828431288 +0000 UTC m=+0.019080491 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:33:39.143 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:38 smithi184 podman[62496]: 2024-02-07 12:33:38.947842195 +0000 UTC m=+0.138491390 container create 653dd44c30002917ef2b46e193268d3844ae204c79e4eb610f5bbdfab4182fce (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, io.buildah.version=1.33.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-02-07T12:33:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:39 smithi184 ceph-mon[53060]: pgmap v259: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 8 peering, 62 active+clean; 360 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.7 MiB/s wr, 359 op/s; 130/1551 objects degraded (8.382%) 2024-02-07T12:33:39.558 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:39 smithi174 ceph-mon[66222]: pgmap v259: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 8 peering, 62 active+clean; 360 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.7 MiB/s wr, 359 op/s; 130/1551 objects degraded (8.382%) 2024-02-07T12:33:40.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 286840 9.67 MB/sec execute 567 sec latency 132.127 ms 2024-02-07T12:33:40.448 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:40 smithi184 podman[62496]: 2024-02-07 12:33:40.053688972 +0000 UTC m=+1.244338173 container init 653dd44c30002917ef2b46e193268d3844ae204c79e4eb610f5bbdfab4182fce (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True) 2024-02-07T12:33:40.449 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:40 smithi184 podman[62496]: 2024-02-07 12:33:40.056954298 +0000 UTC m=+1.247603493 container start 653dd44c30002917ef2b46e193268d3844ae204c79e4eb610f5bbdfab4182fce (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2) 2024-02-07T12:33:40.449 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:40 smithi184 ceph-osd[62515]: -- 172.21.15.184:0/2212485272 <== mon.1 v2:172.21.15.184:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5571998e76c0 con 0x5571998b9000 2024-02-07T12:33:40.449 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:40 smithi184 bash[62496]: 653dd44c30002917ef2b46e193268d3844ae204c79e4eb610f5bbdfab4182fce 2024-02-07T12:33:40.449 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:40 smithi184 systemd[1]: Started Ceph osd.6 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:33:40.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:33:40.600+0000 7fb3d5d9b7c0 -1 Falling back to public interface 2024-02-07T12:33:41.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 287125 9.67 MB/sec execute 568 sec latency 35.761 ms 2024-02-07T12:33:41.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:41 smithi174 ceph-mon[66222]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 362 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.9 MiB/s wr, 397 op/s; 175/1548 objects degraded (11.305%) 2024-02-07T12:33:41.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:41 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:41.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:41 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:41.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:41 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:41 smithi184 ceph-mon[53060]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 362 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.9 MiB/s wr, 397 op/s; 175/1548 objects degraded (11.305%) 2024-02-07T12:33:41.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:41 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:41.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:41 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:41.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:41 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:42.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 287492 9.67 MB/sec execute 569 sec latency 58.690 ms 2024-02-07T12:33:42.375 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:42 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 185/1653 objects degraded (11.192%), 25 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:42.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:42 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 185/1653 objects degraded (11.192%), 25 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:43.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 287769 9.67 MB/sec execute 570 sec latency 45.499 ms 2024-02-07T12:33:43.385 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:43 smithi174 ceph-mon[66222]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 364 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.9 MiB/s wr, 480 op/s; 185/1653 objects degraded (11.192%) 2024-02-07T12:33:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:43 smithi184 ceph-mon[53060]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 364 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.9 MiB/s wr, 480 op/s; 185/1653 objects degraded (11.192%) 2024-02-07T12:33:44.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 288053 9.67 MB/sec execute 571 sec latency 62.260 ms 2024-02-07T12:33:44.932 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:44 smithi174 ceph-mon[66222]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 360 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 419 op/s; 186/1635 objects degraded (11.376%) 2024-02-07T12:33:44.933 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:44 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:44.933 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:44 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:45.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 288344 9.67 MB/sec execute 572 sec latency 54.587 ms 2024-02-07T12:33:45.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:44 smithi184 ceph-mon[53060]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 360 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 419 op/s; 186/1635 objects degraded (11.376%) 2024-02-07T12:33:45.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:44 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:45.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:44 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:46.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 288644 9.65 MB/sec execute 573 sec latency 66.690 ms 2024-02-07T12:33:47.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 288931 9.65 MB/sec execute 574 sec latency 51.506 ms 2024-02-07T12:33:47.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:47 smithi174 ceph-mon[66222]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 366 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 496 op/s; 191/1629 objects degraded (11.725%) 2024-02-07T12:33:47.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:47 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:47.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:47 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:47 smithi184 ceph-mon[53060]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 366 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 496 op/s; 191/1629 objects degraded (11.725%) 2024-02-07T12:33:47.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:47 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:47.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:47 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:48.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 289243 9.65 MB/sec execute 575 sec latency 50.153 ms 2024-02-07T12:33:48.264 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:48 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 191/1614 objects degraded (11.834%), 25 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:48.514 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:48 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 191/1614 objects degraded (11.834%), 25 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:49.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 289640 9.65 MB/sec execute 576 sec latency 42.890 ms 2024-02-07T12:33:49.283 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:49 smithi174 ceph-mon[66222]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 368 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.0 MiB/s wr, 530 op/s; 191/1614 objects degraded (11.834%) 2024-02-07T12:33:49.669 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:49 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:33:49.338+0000 7fb3d5d9b7c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-02-07T12:33:49.669 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:49 smithi184 ceph-mon[53060]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 368 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.0 MiB/s wr, 530 op/s; 191/1614 objects degraded (11.834%) 2024-02-07T12:33:50.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 290003 9.65 MB/sec execute 577 sec latency 59.412 ms 2024-02-07T12:33:51.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 290410 9.66 MB/sec execute 578 sec latency 38.117 ms 2024-02-07T12:33:51.116 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:50 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:33:50.770+0000 7fb3d5d9b7c0 -1 osd.6 138 log_to_monitors true 2024-02-07T12:33:51.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 363 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 437 op/s; 187/1608 objects degraded (11.629%) 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='osd.6 [v2:172.21.15.184:6816/623017400,v1:172.21.15.184:6817/623017400]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:51.325 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:51.326 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:33:51.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 363 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 437 op/s; 187/1608 objects degraded (11.629%) 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='osd.6 [v2:172.21.15.184:6816/623017400,v1:172.21.15.184:6817/623017400]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:33:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:33:52.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 290621 9.65 MB/sec execute 579 sec latency 187.121 ms 2024-02-07T12:33:52.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:33:51 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:33:51.728+0000 7fb3c31f2700 -1 osd.6 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:33:52.384 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:52 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:33:52.384 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:52 smithi174 ceph-mon[66222]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-02-07T12:33:52.384 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:52 smithi174 ceph-mon[66222]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:33:52.384 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:52 smithi174 ceph-mon[66222]: from='osd.6 [v2:172.21.15.184:6816/623017400,v1:172.21.15.184:6817/623017400]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:33:52.384 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:52 smithi174 ceph-mon[66222]: osdmap e141: 8 total, 7 up, 8 in 2024-02-07T12:33:52.384 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:52 smithi174 ceph-mon[66222]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:33:52.384 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:52 smithi174 ceph-mon[66222]: from='osd.6 ' entity='osd.6' 2024-02-07T12:33:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:52 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:33:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:52 smithi184 ceph-mon[53060]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-02-07T12:33:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:52 smithi184 ceph-mon[53060]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-02-07T12:33:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:52 smithi184 ceph-mon[53060]: from='osd.6 [v2:172.21.15.184:6816/623017400,v1:172.21.15.184:6817/623017400]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:33:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:52 smithi184 ceph-mon[53060]: osdmap e141: 8 total, 7 up, 8 in 2024-02-07T12:33:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:52 smithi184 ceph-mon[53060]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:33:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:52 smithi184 ceph-mon[53060]: from='osd.6 ' entity='osd.6' 2024-02-07T12:33:53.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 290907 9.65 MB/sec execute 580 sec latency 337.347 ms 2024-02-07T12:33:53.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:53 smithi174 ceph-mon[66222]: failed to encode map e141 with expected crc 2024-02-07T12:33:53.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:53 smithi174 ceph-mon[66222]: failed to encode map e141 with expected crc 2024-02-07T12:33:53.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:53 smithi174 ceph-mon[66222]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 364 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 536 op/s; 187/1611 objects degraded (11.608%) 2024-02-07T12:33:53.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:53 smithi174 ceph-mon[66222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:33:53.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:53 smithi174 ceph-mon[66222]: osd.6 [v2:172.21.15.184:6816/623017400,v1:172.21.15.184:6817/623017400] boot 2024-02-07T12:33:53.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:53 smithi174 ceph-mon[66222]: osdmap e142: 8 total, 8 up, 8 in 2024-02-07T12:33:53.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:33:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:53 smithi184 ceph-mon[53060]: failed to encode map e141 with expected crc 2024-02-07T12:33:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:53 smithi184 ceph-mon[53060]: failed to encode map e141 with expected crc 2024-02-07T12:33:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:53 smithi184 ceph-mon[53060]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 364 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 536 op/s; 187/1611 objects degraded (11.608%) 2024-02-07T12:33:53.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:53 smithi184 ceph-mon[53060]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:33:53.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:53 smithi184 ceph-mon[53060]: osd.6 [v2:172.21.15.184:6816/623017400,v1:172.21.15.184:6817/623017400] boot 2024-02-07T12:33:53.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:53 smithi184 ceph-mon[53060]: osdmap e142: 8 total, 8 up, 8 in 2024-02-07T12:33:53.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-02-07T12:33:54.060 INFO:teuthology.orchestra.run.smithi174.stdout: 5 290907 9.64 MB/sec execute 581 sec latency 1337.374 ms 2024-02-07T12:33:54.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:54 smithi174 ceph-mon[66222]: failed to encode map e142 with expected crc 2024-02-07T12:33:54.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:54 smithi174 ceph-mon[66222]: osdmap e143: 8 total, 8 up, 8 in 2024-02-07T12:33:54.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:54 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 187/1611 objects degraded (11.608%), 25 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:54.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:54 smithi184 ceph-mon[53060]: failed to encode map e142 with expected crc 2024-02-07T12:33:54.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:54 smithi184 ceph-mon[53060]: osdmap e143: 8 total, 8 up, 8 in 2024-02-07T12:33:54.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:54 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 187/1611 objects degraded (11.608%), 25 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:55.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 290907 9.62 MB/sec execute 582 sec latency 2337.445 ms 2024-02-07T12:33:55.524 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:55 smithi174 ceph-mon[66222]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:55 smithi174 ceph-mon[66222]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:55 smithi174 ceph-mon[66222]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:55 smithi174 ceph-mon[66222]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:55 smithi174 ceph-mon[66222]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:55 smithi174 ceph-mon[66222]: pgmap v270: 97 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 9 active+undersized, 21 active+undersized+degraded, 65 active+clean; 365 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 447 op/s; 174/1605 objects degraded (10.841%); 682 B/s, 0 objects/s recovering 2024-02-07T12:33:55.525 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:55 smithi174 ceph-mon[66222]: osdmap e144: 8 total, 8 up, 8 in 2024-02-07T12:33:55.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:55 smithi184 ceph-mon[53060]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:55 smithi184 ceph-mon[53060]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:55 smithi184 ceph-mon[53060]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:55 smithi184 ceph-mon[53060]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:55 smithi184 ceph-mon[53060]: failed to encode map e143 with expected crc 2024-02-07T12:33:55.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:55 smithi184 ceph-mon[53060]: pgmap v270: 97 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 9 active+undersized, 21 active+undersized+degraded, 65 active+clean; 365 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 447 op/s; 174/1605 objects degraded (10.841%); 682 B/s, 0 objects/s recovering 2024-02-07T12:33:55.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:55 smithi184 ceph-mon[53060]: osdmap e144: 8 total, 8 up, 8 in 2024-02-07T12:33:56.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 291068 9.61 MB/sec execute 583 sec latency 2833.755 ms 2024-02-07T12:33:56.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:56 smithi174 ceph-mon[66222]: failed to encode map e144 with expected crc 2024-02-07T12:33:56.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:56 smithi174 ceph-mon[66222]: failed to encode map e144 with expected crc 2024-02-07T12:33:56.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:56 smithi174 ceph-mon[66222]: failed to encode map e144 with expected crc 2024-02-07T12:33:56.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:56 smithi184 ceph-mon[53060]: failed to encode map e144 with expected crc 2024-02-07T12:33:56.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:56 smithi184 ceph-mon[53060]: failed to encode map e144 with expected crc 2024-02-07T12:33:56.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:56 smithi184 ceph-mon[53060]: failed to encode map e144 with expected crc 2024-02-07T12:33:57.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 291396 9.61 MB/sec execute 584 sec latency 46.627 ms 2024-02-07T12:33:57.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:57 smithi174 ceph-mon[66222]: pgmap v272: 97 pgs: 3 activating+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovery_wait+degraded, 4 active+undersized, 17 active+undersized+degraded, 70 active+clean; 366 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 404 op/s; 193/1650 objects degraded (11.697%); 842 B/s, 0 objects/s recovering 2024-02-07T12:33:57.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:57 smithi174 ceph-mon[66222]: failed to encode map e145 with expected crc 2024-02-07T12:33:57.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:57 smithi174 ceph-mon[66222]: osdmap e145: 8 total, 8 up, 8 in 2024-02-07T12:33:57.599 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:57 smithi184 ceph-mon[53060]: pgmap v272: 97 pgs: 3 activating+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovery_wait+degraded, 4 active+undersized, 17 active+undersized+degraded, 70 active+clean; 366 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 404 op/s; 193/1650 objects degraded (11.697%); 842 B/s, 0 objects/s recovering 2024-02-07T12:33:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:57 smithi184 ceph-mon[53060]: failed to encode map e145 with expected crc 2024-02-07T12:33:57.600 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:57 smithi184 ceph-mon[53060]: osdmap e145: 8 total, 8 up, 8 in 2024-02-07T12:33:58.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 291639 9.60 MB/sec execute 585 sec latency 810.071 ms 2024-02-07T12:33:58.346 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:58 smithi174 ceph-mon[66222]: failed to encode map e146 with expected crc 2024-02-07T12:33:58.346 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:58 smithi174 ceph-mon[66222]: osdmap e146: 8 total, 8 up, 8 in 2024-02-07T12:33:58.346 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:58 smithi174 ceph-mon[66222]: failed to encode map e146 with expected crc 2024-02-07T12:33:58.611 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:58 smithi184 ceph-mon[53060]: failed to encode map e146 with expected crc 2024-02-07T12:33:58.611 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:58 smithi184 ceph-mon[53060]: osdmap e146: 8 total, 8 up, 8 in 2024-02-07T12:33:58.611 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:58 smithi184 ceph-mon[53060]: failed to encode map e146 with expected crc 2024-02-07T12:33:59.063 INFO:teuthology.orchestra.run.smithi174.stdout: 5 291974 9.60 MB/sec execute 586 sec latency 1054.935 ms 2024-02-07T12:33:59.286 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:59 smithi174 ceph-mon[66222]: pgmap v275: 97 pgs: 3 peering, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 366 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.5 MiB/s wr, 284 op/s; 99/1644 objects degraded (6.022%); 1.4 MiB/s, 58 keys/s, 19 objects/s recovering 2024-02-07T12:33:59.286 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:59 smithi174 ceph-mon[66222]: failed to encode map e147 with expected crc 2024-02-07T12:33:59.286 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:59 smithi174 ceph-mon[66222]: osdmap e147: 8 total, 8 up, 8 in 2024-02-07T12:33:59.286 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:59 smithi174 ceph-mon[66222]: failed to encode map e147 with expected crc 2024-02-07T12:33:59.286 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:33:59 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 99/1644 objects degraded (6.022%), 5 pgs degraded (PG_DEGRADED) 2024-02-07T12:33:59.577 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:33:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:59 smithi184 ceph-mon[53060]: pgmap v275: 97 pgs: 3 peering, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 366 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.5 MiB/s wr, 284 op/s; 99/1644 objects degraded (6.022%); 1.4 MiB/s, 58 keys/s, 19 objects/s recovering 2024-02-07T12:33:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:59 smithi184 ceph-mon[53060]: failed to encode map e147 with expected crc 2024-02-07T12:33:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:59 smithi184 ceph-mon[53060]: osdmap e147: 8 total, 8 up, 8 in 2024-02-07T12:33:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:59 smithi184 ceph-mon[53060]: failed to encode map e147 with expected crc 2024-02-07T12:33:59.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:33:59 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 99/1644 objects degraded (6.022%), 5 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:00.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 292396 9.60 MB/sec execute 587 sec latency 43.750 ms 2024-02-07T12:34:00.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:00 smithi184 ceph-mon[53060]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:00.489 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:34:00.489 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (9m) 2m ago 18m 22.9M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:34:00.489 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (5m) 2m ago 18m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:34:00.489 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (5m) 15s ago 17m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (8m) 2m ago 17m 53.2M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (12m) 2m ago 12m 38.7M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (12m) 15s ago 12m 14.3M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (8m) 2m ago 19m 507M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (7m) 15s ago 17m 451M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (6m) 2m ago 19m 50.1M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (6m) 15s ago 17m 43.1M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (12m) 2m ago 12m 274M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (11m) 15s ago 11m 87.1M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 2m ago 17m 17.1M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (8m) 15s ago 17m 21.2M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (4m) 2m ago 16m 334M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (3m) 2m ago 16m 289M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:34:00.490 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (3m) 2m ago 15m 235M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:34:00.491 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (2m) 2m ago 15m 11.8M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:34:00.491 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (96s) 15s ago 15m 309M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:34:00.491 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (58s) 15s ago 14m 261M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:34:00.491 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (21s) 15s ago 14m 11.8M 2704M 19.0.0-1252-g44458cce 79957b372307 653dd44c3000 2024-02-07T12:34:00.491 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (13m) 15s ago 13m 667M 2704M 17.2.0 e1d6a67b021e b19d0bdc4954 2024-02-07T12:34:00.491 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (7m) 2m ago 17m 68.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:34:00.491 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:00 smithi174 ceph-mon[66222]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:01.039 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 7 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:34:01.040 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:34:01.041 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:34:01.041 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-02-07T12:34:01.041 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 11 2024-02-07T12:34:01.041 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:34:01.041 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:34:01.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 292706 9.59 MB/sec execute 588 sec latency 57.358 ms 2024-02-07T12:34:01.511 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:01 smithi174 ceph-mon[66222]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:01.511 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:01 smithi174 ceph-mon[66222]: pgmap v277: 97 pgs: 3 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 361 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.0 MiB/s wr, 264 op/s; 26/1635 objects degraded (1.590%); 1.5 MiB/s, 47 keys/s, 33 objects/s recovering 2024-02-07T12:34:01.511 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:01 smithi174 ceph-mon[66222]: from='client.44325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:01.511 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:01 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/218877676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "13/23 daemons upgraded", 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading osd daemons", 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:34:01.516 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:34:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:01 smithi184 ceph-mon[53060]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:01 smithi184 ceph-mon[53060]: pgmap v277: 97 pgs: 3 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 361 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.0 MiB/s wr, 264 op/s; 26/1635 objects degraded (1.590%); 1.5 MiB/s, 47 keys/s, 33 objects/s recovering 2024-02-07T12:34:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:01 smithi184 ceph-mon[53060]: from='client.44325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:01 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/218877676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:02.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 293085 9.60 MB/sec execute 589 sec latency 44.767 ms 2024-02-07T12:34:02.066 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_WARN Degraded data redundancy: 26/1635 objects degraded (1.590%), 2 pgs degraded 2024-02-07T12:34:02.066 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 26/1635 objects degraded (1.590%), 2 pgs degraded 2024-02-07T12:34:02.066 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.a is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-02-07T12:34:02.066 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.11 is active+recovering+undersized+degraded+remapped, acting [7,4] 2024-02-07T12:34:02.531 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:02 smithi174 ceph-mon[66222]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:02.532 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:02 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/714721018' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:34:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:02 smithi184 ceph-mon[53060]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:02.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:02 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/714721018' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:34:03.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 293423 9.59 MB/sec execute 590 sec latency 40.702 ms 2024-02-07T12:34:03.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:03 smithi174 ceph-mon[66222]: pgmap v278: 97 pgs: 97 active+clean; 357 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.2 MiB/s wr, 397 op/s; 3.5 MiB/s, 42 keys/s, 38 objects/s recovering 2024-02-07T12:34:03.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:03 smithi174 ceph-mon[66222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1635 objects degraded (1.590%), 2 pgs degraded) 2024-02-07T12:34:03.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:03 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:34:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:03 smithi184 ceph-mon[53060]: pgmap v278: 97 pgs: 97 active+clean; 357 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.2 MiB/s wr, 397 op/s; 3.5 MiB/s, 42 keys/s, 38 objects/s recovering 2024-02-07T12:34:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:03 smithi184 ceph-mon[53060]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1635 objects degraded (1.590%), 2 pgs degraded) 2024-02-07T12:34:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:03 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:34:04.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 293756 9.60 MB/sec execute 591 sec latency 56.241 ms 2024-02-07T12:34:04.876 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:04 smithi174 ceph-mon[66222]: pgmap v279: 97 pgs: 97 active+clean; 360 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.5 MiB/s wr, 437 op/s; 2.7 MiB/s, 24 objects/s recovering 2024-02-07T12:34:05.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 294036 9.59 MB/sec execute 592 sec latency 48.038 ms 2024-02-07T12:34:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:04 smithi184 ceph-mon[53060]: pgmap v279: 97 pgs: 97 active+clean; 360 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.5 MiB/s wr, 437 op/s; 2.7 MiB/s, 24 objects/s recovering 2024-02-07T12:34:06.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 294305 9.59 MB/sec execute 593 sec latency 52.946 ms 2024-02-07T12:34:06.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:06.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:06.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:34:06.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:06.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:06.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:34:07.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 294598 9.59 MB/sec execute 594 sec latency 53.337 ms 2024-02-07T12:34:07.202 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:07 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:34:07.202 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:07 smithi174 ceph-mon[66222]: Upgrade: osd.7 is safe to restart 2024-02-07T12:34:07.203 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:07 smithi174 ceph-mon[66222]: pgmap v280: 97 pgs: 97 active+clean; 360 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.1 MiB/s wr, 324 op/s; 1.7 MiB/s, 17 objects/s recovering 2024-02-07T12:34:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:07 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-02-07T12:34:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:07 smithi184 ceph-mon[53060]: Upgrade: osd.7 is safe to restart 2024-02-07T12:34:07.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:07 smithi184 ceph-mon[53060]: pgmap v280: 97 pgs: 97 active+clean; 360 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.1 MiB/s wr, 324 op/s; 1.7 MiB/s, 17 objects/s recovering 2024-02-07T12:34:08.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 294885 9.59 MB/sec execute 595 sec latency 53.499 ms 2024-02-07T12:34:09.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 295198 9.59 MB/sec execute 596 sec latency 46.444 ms 2024-02-07T12:34:09.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:09 smithi174 ceph-mon[66222]: Upgrade: Updating osd.7 2024-02-07T12:34:09.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:09.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:34:09.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:09.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:09 smithi174 ceph-mon[66222]: Deploying daemon osd.7 on smithi184 2024-02-07T12:34:09.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:09 smithi174 ceph-mon[66222]: pgmap v281: 97 pgs: 97 active+clean; 361 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 449 op/s; 1.5 MiB/s, 15 objects/s recovering 2024-02-07T12:34:09.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:09 smithi184 ceph-mon[53060]: Upgrade: Updating osd.7 2024-02-07T12:34:09.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:09.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-02-07T12:34:09.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:09.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:09 smithi184 ceph-mon[53060]: Deploying daemon osd.7 on smithi184 2024-02-07T12:34:09.413 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:09 smithi184 ceph-mon[53060]: pgmap v281: 97 pgs: 97 active+clean; 361 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 449 op/s; 1.5 MiB/s, 15 objects/s recovering 2024-02-07T12:34:10.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 295412 9.58 MB/sec execute 597 sec latency 91.853 ms 2024-02-07T12:34:11.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 295700 9.57 MB/sec execute 598 sec latency 57.808 ms 2024-02-07T12:34:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:11 smithi174 ceph-mon[66222]: pgmap v282: 97 pgs: 97 active+clean; 360 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 407 op/s; 1.3 MiB/s, 12 objects/s recovering 2024-02-07T12:34:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:11 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:11.635 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:11 smithi184 ceph-mon[53060]: pgmap v282: 97 pgs: 97 active+clean; 360 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 407 op/s; 1.3 MiB/s, 12 objects/s recovering 2024-02-07T12:34:11.636 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:11 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:12.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 295941 9.56 MB/sec execute 599 sec latency 75.592 ms 2024-02-07T12:34:12.448 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:12 smithi184 systemd[1]: Stopping Ceph osd.7 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:34:13.061 INFO:teuthology.orchestra.run.smithi174.stdout: 5 cleanup 600 sec 2024-02-07T12:34:13.366 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:13 smithi174 ceph-mon[66222]: pgmap v283: 97 pgs: 97 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 522 op/s; 1.1 MiB/s, 4 objects/s recovering 2024-02-07T12:34:13.393 INFO:teuthology.orchestra.run.smithi174.stdout: 0 cleanup 600 sec 2024-02-07T12:34:13.393 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:34:13.393 INFO:teuthology.orchestra.run.smithi174.stdout: Operation Count AvgLat MaxLat 2024-02-07T12:34:13.393 INFO:teuthology.orchestra.run.smithi174.stdout: ---------------------------------------- 2024-02-07T12:34:13.393 INFO:teuthology.orchestra.run.smithi174.stdout: NTCreateX 183653 4.664 2833.748 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Close 134819 0.135 10.780 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Rename 7788 9.790 188.649 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Unlink 37176 7.240 306.015 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Qpathinfo 166586 1.180 1048.184 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Qfileinfo 29056 0.008 0.797 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Qfsinfo 30577 0.637 45.761 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Sfileinfo 14975 13.483 2800.119 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Find 64412 0.942 12.294 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: WriteX 90816 14.254 2799.362 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: ReadX 288324 0.010 3.936 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: LockX 600 0.367 2.564 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: UnlockX 600 0.323 3.329 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: Flush 12878 0.002 0.020 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:34:13.394 INFO:teuthology.orchestra.run.smithi174.stdout:Throughput 9.56328 MB/sec 5 clients 5 procs max_latency=2833.755 ms 2024-02-07T12:34:13.396 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-02-07T12:34:13.435 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-02-07T12:34:13.560 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-02-07T12:34:13.606 INFO:teuthology.orchestra.run.smithi174.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-02-07T12:34:13.606 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:34:13.606 INFO:teuthology.orchestra.run.smithi174.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-02-07T12:34:13.607 INFO:teuthology.orchestra.run.smithi174.stdout:3 of 5 processes prepared for launch 0 sec 2024-02-07T12:34:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:13 smithi184 ceph-mon[53060]: pgmap v283: 97 pgs: 97 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 522 op/s; 1.1 MiB/s, 4 objects/s recovering 2024-02-07T12:34:13.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:13 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[40290]: 2024-02-07T12:34:13.288+0000 7fcad7473700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:34:13.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:13 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[40290]: 2024-02-07T12:34:13.288+0000 7fcad7473700 -1 osd.7 147 *** Got signal Terminated *** 2024-02-07T12:34:13.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:13 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[40290]: 2024-02-07T12:34:13.288+0000 7fcad7473700 -1 osd.7 147 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:34:13.706 INFO:teuthology.orchestra.run.smithi174.stdout:5 of 5 processes prepared for launch 0 sec 2024-02-07T12:34:13.707 INFO:teuthology.orchestra.run.smithi174.stdout:releasing clients 2024-02-07T12:34:14.490 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:14 smithi174 ceph-mon[66222]: osd.7 marked itself down and dead 2024-02-07T12:34:14.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:14 smithi184 ceph-mon[53060]: osd.7 marked itself down and dead 2024-02-07T12:34:15.258 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:14 smithi184 podman[64277]: 2024-02-07 12:34:14.989253567 +0000 UTC m=+2.666381404 container died b19d0bdc49545ab8aaf2de5fc869f6f279c347b9f0b2c09bfa0cb48d5e22628d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , release=754, io.buildah.version=1.19.8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, ceph=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vendor=Red Hat, Inc.) 2024-02-07T12:34:15.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:15 smithi174 ceph-mon[66222]: pgmap v284: 97 pgs: 97 active+clean; 358 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 442 op/s 2024-02-07T12:34:15.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:15 smithi174 ceph-mon[66222]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:34:15.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:15 smithi174 ceph-mon[66222]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-02-07T12:34:15.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:15 smithi174 ceph-mon[66222]: osdmap e148: 8 total, 7 up, 8 in 2024-02-07T12:34:15.652 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:15 smithi184 ceph-mon[53060]: pgmap v284: 97 pgs: 97 active+clean; 358 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 442 op/s 2024-02-07T12:34:15.653 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:15 smithi184 ceph-mon[53060]: Health check failed: 1 osds down (OSD_DOWN) 2024-02-07T12:34:15.653 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:15 smithi184 ceph-mon[53060]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-02-07T12:34:15.653 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:15 smithi184 ceph-mon[53060]: osdmap e148: 8 total, 7 up, 8 in 2024-02-07T12:34:15.653 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:15 smithi184 podman[64277]: 2024-02-07 12:34:15.348942344 +0000 UTC m=+3.026070188 container cleanup b19d0bdc49545ab8aaf2de5fc869f6f279c347b9f0b2c09bfa0cb48d5e22628d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-type=git, com.redhat.component=centos-stream-container, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.expose-services=, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-02-07T12:34:15.654 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:15 smithi184 bash[64277]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7 2024-02-07T12:34:15.707 INFO:teuthology.orchestra.run.smithi174.stdout: 5 193 5.98 MB/sec execute 1 sec latency 925.709 ms 2024-02-07T12:34:15.948 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:15 smithi184 podman[64329]: 2024-02-07 12:34:15.649020484 +0000 UTC m=+0.656851954 container remove b19d0bdc49545ab8aaf2de5fc869f6f279c347b9f0b2c09bfa0cb48d5e22628d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, vcs-type=git, architecture=x86_64, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc.) 2024-02-07T12:34:16.280 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:16 smithi184 podman[64409]: 2024-02-07 12:34:15.90528044 +0000 UTC m=+0.019881812 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:34:16.281 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:16 smithi184 podman[64409]: 2024-02-07 12:34:16.025118467 +0000 UTC m=+0.139719838 container create 8db3691335f76a750ce859887f23ec91540e072c0a035f82726270d786f473fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2) 2024-02-07T12:34:16.326 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:16 smithi174 ceph-mon[66222]: osdmap e149: 8 total, 7 up, 8 in 2024-02-07T12:34:16.651 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:16 smithi184 ceph-mon[53060]: osdmap e149: 8 total, 7 up, 8 in 2024-02-07T12:34:16.651 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:16 smithi184 podman[64409]: 2024-02-07 12:34:16.278732111 +0000 UTC m=+0.393333485 container init 8db3691335f76a750ce859887f23ec91540e072c0a035f82726270d786f473fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:34:16.651 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:16 smithi184 podman[64409]: 2024-02-07 12:34:16.283666005 +0000 UTC m=+0.398267384 container start 8db3691335f76a750ce859887f23ec91540e072c0a035f82726270d786f473fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-02-07T12:34:16.651 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:16 smithi184 podman[64409]: 2024-02-07 12:34:16.386829221 +0000 UTC m=+0.501430598 container attach 8db3691335f76a750ce859887f23ec91540e072c0a035f82726270d786f473fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-02-07T12:34:16.708 INFO:teuthology.orchestra.run.smithi174.stdout: 5 323 13.26 MB/sec execute 2 sec latency 57.264 ms 2024-02-07T12:34:16.948 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:16 smithi184 podman[64409]: 2024-02-07 12:34:16.650672454 +0000 UTC m=+0.765273826 container died 8db3691335f76a750ce859887f23ec91540e072c0a035f82726270d786f473fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux ) 2024-02-07T12:34:17.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:17 smithi174 ceph-mon[66222]: pgmap v287: 97 pgs: 8 peering, 8 stale+active+clean, 81 active+clean; 365 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 307 op/s 2024-02-07T12:34:17.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:17 smithi174 ceph-mon[66222]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:34:17.419 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:17 smithi184 ceph-mon[53060]: pgmap v287: 97 pgs: 8 peering, 8 stale+active+clean, 81 active+clean; 365 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 307 op/s 2024-02-07T12:34:17.419 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:17 smithi184 ceph-mon[53060]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:34:17.686 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:17 smithi184 podman[64435]: 2024-02-07 12:34:17.41729478 +0000 UTC m=+0.761376296 container remove 8db3691335f76a750ce859887f23ec91540e072c0a035f82726270d786f473fb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2) 2024-02-07T12:34:17.686 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:17 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.7.service: Deactivated successfully. 2024-02-07T12:34:17.686 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:17 smithi184 systemd[1]: Stopped Ceph osd.7 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:34:17.686 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:17 smithi184 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.7.service: Consumed 8min 15.682s CPU time. 2024-02-07T12:34:17.707 INFO:teuthology.orchestra.run.smithi174.stdout: 5 461 16.04 MB/sec execute 3 sec latency 39.891 ms 2024-02-07T12:34:17.945 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:17 smithi184 systemd[1]: Starting Ceph osd.7 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:34:17.945 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:17 smithi184 podman[64539]: 2024-02-07 12:34:17.832739616 +0000 UTC m=+0.023466082 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:34:17.945 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:17 smithi184 podman[64539]: 2024-02-07 12:34:17.943517507 +0000 UTC m=+0.134243956 container create d5860138ba6ba2def19c90f7ea1258a99f1d2d1a483888d5e7d403e083f2634d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-02-07T12:34:18.387 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:18 smithi174 ceph-mon[66222]: Health check failed: Degraded data redundancy: 60/1305 objects degraded (4.598%), 15 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:18.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:18 smithi184 ceph-mon[53060]: Health check failed: Degraded data redundancy: 60/1305 objects degraded (4.598%), 15 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:18.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 podman[64539]: 2024-02-07 12:34:18.196796729 +0000 UTC m=+0.387523180 container init d5860138ba6ba2def19c90f7ea1258a99f1d2d1a483888d5e7d403e083f2634d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, org.label-schema.license=GPLv2) 2024-02-07T12:34:18.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 podman[64539]: 2024-02-07 12:34:18.200844296 +0000 UTC m=+0.391570753 container start d5860138ba6ba2def19c90f7ea1258a99f1d2d1a483888d5e7d403e083f2634d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-02-07T12:34:18.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 podman[64539]: 2024-02-07 12:34:18.30549744 +0000 UTC m=+0.496223891 container attach d5860138ba6ba2def19c90f7ea1258a99f1d2d1a483888d5e7d403e083f2634d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate, ceph=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-02-07T12:34:18.707 INFO:teuthology.orchestra.run.smithi174.stdout: 5 663 18.25 MB/sec execute 4 sec latency 152.736 ms 2024-02-07T12:34:19.069 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate[64554]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate[64554]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 bash[64539]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 bash[64539]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 bash[64539]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 bash[64539]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 bash[64539]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 bash[64539]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 bash[64539]: --> ceph-volume raw activate successful for osd ID: 7 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate[64554]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate[64554]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate[64554]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-02-07T12:34:19.070 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate[64554]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-02-07T12:34:19.071 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate[64554]: --> ceph-volume raw activate successful for osd ID: 7 2024-02-07T12:34:19.071 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:18 smithi184 podman[64539]: 2024-02-07 12:34:18.75444632 +0000 UTC m=+0.945172788 container died d5860138ba6ba2def19c90f7ea1258a99f1d2d1a483888d5e7d403e083f2634d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-02-07T12:34:19.433 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:19 smithi174 ceph-mon[66222]: pgmap v288: 97 pgs: 2 active+undersized, 15 active+undersized+degraded, 8 peering, 5 stale+active+clean, 67 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 419 op/s; 60/1305 objects degraded (4.598%) 2024-02-07T12:34:19.591 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:19 smithi184 ceph-mon[53060]: pgmap v288: 97 pgs: 2 active+undersized, 15 active+undersized+degraded, 8 peering, 5 stale+active+clean, 67 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 419 op/s; 60/1305 objects degraded (4.598%) 2024-02-07T12:34:19.591 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:19 smithi184 podman[64685]: 2024-02-07 12:34:19.368865976 +0000 UTC m=+0.607817854 container remove d5860138ba6ba2def19c90f7ea1258a99f1d2d1a483888d5e7d403e083f2634d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS) 2024-02-07T12:34:19.707 INFO:teuthology.orchestra.run.smithi174.stdout: 5 cleanup 5 sec 2024-02-07T12:34:19.848 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:19 smithi184 podman[64747]: 2024-02-07 12:34:19.494125667 +0000 UTC m=+0.020725905 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:34:19.848 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:19 smithi184 podman[64747]: 2024-02-07 12:34:19.594934547 +0000 UTC m=+0.121534783 container create 0445d4324ff0ca8bb5d28e642a6bf5bbd5b133f33c733fec2c6bb2ffb2899801 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-02-07T12:34:19.848 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:19 smithi184 podman[64747]: 2024-02-07 12:34:19.822848822 +0000 UTC m=+0.349449072 container init 0445d4324ff0ca8bb5d28e642a6bf5bbd5b133f33c733fec2c6bb2ffb2899801 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-02-07T12:34:19.848 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:19 smithi184 podman[64747]: 2024-02-07 12:34:19.825992579 +0000 UTC m=+0.352592817 container start 0445d4324ff0ca8bb5d28e642a6bf5bbd5b133f33c733fec2c6bb2ffb2899801 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131) 2024-02-07T12:34:19.890 INFO:teuthology.orchestra.run.smithi174.stdout: 0 cleanup 5 sec 2024-02-07T12:34:19.890 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:34:19.890 INFO:teuthology.orchestra.run.smithi174.stdout: Operation Count AvgLat MaxLat 2024-02-07T12:34:19.890 INFO:teuthology.orchestra.run.smithi174.stdout: ---------------------------------------- 2024-02-07T12:34:19.890 INFO:teuthology.orchestra.run.smithi174.stdout: NTCreateX 586 10.612 64.490 2024-02-07T12:34:19.890 INFO:teuthology.orchestra.run.smithi174.stdout: Close 551 10.734 152.731 2024-02-07T12:34:19.890 INFO:teuthology.orchestra.run.smithi174.stdout: Rename 19 23.422 67.926 2024-02-07T12:34:19.890 INFO:teuthology.orchestra.run.smithi174.stdout: Unlink 46 2.605 14.038 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: Qpathinfo 148 1.036 12.697 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: Qfileinfo 338 31.084 925.702 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: Qfsinfo 91 0.420 1.635 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: Sfileinfo 82 11.237 23.816 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: Find 121 0.797 2.415 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: WriteX 1641 0.026 0.081 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: ReadX 418 0.017 0.330 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: Flush 112 13.319 38.329 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:34:19.891 INFO:teuthology.orchestra.run.smithi174.stdout:Throughput 18.2522 MB/sec 5 clients 5 procs max_latency=925.709 ms 2024-02-07T12:34:19.892 DEBUG:teuthology.parallel:result is None 2024-02-07T12:34:20.198 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:19 smithi184 ceph-osd[64766]: -- 172.21.15.184:0/3918499390 <== mon.1 v2:172.21.15.184:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560e35f616c0 con 0x560e35f33000 2024-02-07T12:34:20.198 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:19 smithi184 bash[64747]: 0445d4324ff0ca8bb5d28e642a6bf5bbd5b133f33c733fec2c6bb2ffb2899801 2024-02-07T12:34:20.198 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:19 smithi184 systemd[1]: Started Ceph osd.7 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:34:20.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:20 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:34:20.379+0000 7f78bfcfc7c0 -1 Falling back to public interface 2024-02-07T12:34:21.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:21.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:21.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:21.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:20 smithi184 ceph-mon[53060]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s wr, 269 op/s; 183/1125 objects degraded (16.267%) 2024-02-07T12:34:21.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:21.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:21.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:21.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:21.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:21.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:20 smithi174 ceph-mon[66222]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s wr, 269 op/s; 183/1125 objects degraded (16.267%) 2024-02-07T12:34:21.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:21.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:22.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:21 smithi174 ceph-mon[66222]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-02-07T12:34:22.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:21 smithi184 ceph-mon[53060]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-02-07T12:34:23.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:22 smithi184 ceph-mon[53060]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 15 MiB/s wr, 436 op/s; 183/1206 objects degraded (15.174%) 2024-02-07T12:34:23.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:22 smithi174 ceph-mon[66222]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 15 MiB/s wr, 436 op/s; 183/1206 objects degraded (15.174%) 2024-02-07T12:34:24.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:23 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 183/1206 objects degraded (15.174%), 35 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:24.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:23 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:24.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:23 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:24.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:23 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 183/1206 objects degraded (15.174%), 35 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:24.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:23 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:24.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:23 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:25.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:24 smithi174 ceph-mon[66222]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 320 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 12 MiB/s wr, 349 op/s; 165/1083 objects degraded (15.235%) 2024-02-07T12:34:25.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:24 smithi184 ceph-mon[53060]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 320 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 12 MiB/s wr, 349 op/s; 165/1083 objects degraded (15.235%) 2024-02-07T12:34:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:26 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:26.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:26 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:26.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:26 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:26.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:26 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:27.378 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:27 smithi184 ceph-mon[53060]: pgmap v292: 97 pgs: 9 active+undersized, 33 active+undersized+degraded, 55 active+clean; 282 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 9.7 MiB/s wr, 318 op/s; 101/717 objects degraded (14.086%) 2024-02-07T12:34:27.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:27 smithi174 ceph-mon[66222]: pgmap v292: 97 pgs: 9 active+undersized, 33 active+undersized+degraded, 55 active+clean; 282 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 9.7 MiB/s wr, 318 op/s; 101/717 objects degraded (14.086%) 2024-02-07T12:34:28.432 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:28 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:34:28.294+0000 7f78bfcfc7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: pgmap v293: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 248 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 8.8 MiB/s wr, 296 op/s; 88/459 objects degraded (19.172%) 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 88/459 objects degraded (19.172%), 32 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-07T12:34:29.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-02-07T12:34:29.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: pgmap v293: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 248 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 8.8 MiB/s wr, 296 op/s; 88/459 objects degraded (19.172%) 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 88/459 objects degraded (19.172%), 32 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:29.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-02-07T12:34:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-02-07T12:34:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-02-07T12:34:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-02-07T12:34:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-02-07T12:34:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-02-07T12:34:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-02-07T12:34:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-02-07T12:34:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-02-07T12:34:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-02-07T12:34:29.998 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:34:29.731+0000 7f78bfcfc7c0 -1 osd.7 147 log_to_monitors true 2024-02-07T12:34:29.998 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:34:29 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:34:29.995+0000 7f78b115b700 -1 osd.7 147 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-02-07T12:34:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all osd 2024-02-07T12:34:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: Upgrade: Setting require_osd_release to 19 squid 2024-02-07T12:34:30.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: from='osd.7 [v2:172.21.15.184:6824/3941008724,v1:172.21.15.184:6825/3941008724]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:34:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:34:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-02-07T12:34:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-02-07T12:34:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:34:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: from='osd.7 [v2:172.21.15.184:6824/3941008724,v1:172.21.15.184:6825/3941008724]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:34:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: osdmap e150: 8 total, 7 up, 8 in 2024-02-07T12:34:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:34:30.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi174.vkrzkp"]}]: dispatch 2024-02-07T12:34:30.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all osd 2024-02-07T12:34:30.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: Upgrade: Setting require_osd_release to 19 squid 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: from='osd.7 [v2:172.21.15.184:6824/3941008724,v1:172.21.15.184:6825/3941008724]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: from='osd.7 [v2:172.21.15.184:6824/3941008724,v1:172.21.15.184:6825/3941008724]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: osdmap e150: 8 total, 7 up, 8 in 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi184", "root=default"]}]: dispatch 2024-02-07T12:34:30.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi174.vkrzkp"]}]: dispatch 2024-02-07T12:34:31.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:31 smithi184 ceph-mon[53060]: Upgrade: It appears safe to stop mds.foofs.smithi174.vkrzkp 2024-02-07T12:34:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:31 smithi184 ceph-mon[53060]: pgmap v295: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.0 MiB/s wr, 170 op/s; 82/441 objects degraded (18.594%) 2024-02-07T12:34:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:31 smithi184 ceph-mon[53060]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:34:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:31 smithi184 ceph-mon[53060]: osd.7 [v2:172.21.15.184:6824/3941008724,v1:172.21.15.184:6825/3941008724] boot 2024-02-07T12:34:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:31 smithi184 ceph-mon[53060]: osdmap e151: 8 total, 8 up, 8 in 2024-02-07T12:34:31.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:34:31.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:31 smithi174 ceph-mon[66222]: Upgrade: It appears safe to stop mds.foofs.smithi174.vkrzkp 2024-02-07T12:34:31.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:31 smithi174 ceph-mon[66222]: pgmap v295: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.0 MiB/s wr, 170 op/s; 82/441 objects degraded (18.594%) 2024-02-07T12:34:31.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:31 smithi174 ceph-mon[66222]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-02-07T12:34:31.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:31 smithi174 ceph-mon[66222]: osd.7 [v2:172.21.15.184:6824/3941008724,v1:172.21.15.184:6825/3941008724] boot 2024-02-07T12:34:31.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:31 smithi174 ceph-mon[66222]: osdmap e151: 8 total, 8 up, 8 in 2024-02-07T12:34:31.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-02-07T12:34:32.444 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:34:32.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:32 smithi184 ceph-mon[53060]: Upgrade: Updating mds.foofs.smithi174.vkrzkp 2024-02-07T12:34:32.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:32 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:32.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:32 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.vkrzkp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:34:32.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:32 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:32.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:32 smithi184 ceph-mon[53060]: Deploying daemon mds.foofs.smithi174.vkrzkp on smithi174 2024-02-07T12:34:32.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:32 smithi184 ceph-mon[53060]: osdmap e152: 8 total, 8 up, 8 in 2024-02-07T12:34:32.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:32 smithi184 ceph-mon[53060]: pgmap v298: 97 pgs: 2 activating+degraded, 1 remapped+peering, 7 peering, 7 active+undersized, 21 active+undersized+degraded, 59 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 53 KiB/s wr, 21 op/s; 84/441 objects degraded (19.048%) 2024-02-07T12:34:32.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:32 smithi184 ceph-mon[53060]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:33.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:32 smithi174 ceph-mon[66222]: Upgrade: Updating mds.foofs.smithi174.vkrzkp 2024-02-07T12:34:33.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:33.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.vkrzkp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:34:33.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:33.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:32 smithi174 ceph-mon[66222]: Deploying daemon mds.foofs.smithi174.vkrzkp on smithi174 2024-02-07T12:34:33.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:32 smithi174 ceph-mon[66222]: osdmap e152: 8 total, 8 up, 8 in 2024-02-07T12:34:33.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:32 smithi174 ceph-mon[66222]: pgmap v298: 97 pgs: 2 activating+degraded, 1 remapped+peering, 7 peering, 7 active+undersized, 21 active+undersized+degraded, 59 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 53 KiB/s wr, 21 op/s; 84/441 objects degraded (19.048%) 2024-02-07T12:34:33.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:32 smithi174 ceph-mon[66222]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (9m) 2m ago 18m 22.9M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (5m) 2m ago 18m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (5m) 9s ago 18m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (9m) 2m ago 18m 53.2M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (12m) 2m ago 12m 38.7M - 17.2.0 e1d6a67b021e 73a8b14ca64f 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (12m) 9s ago 12m 14.4M - 17.2.0 e1d6a67b021e f6548204c4bd 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (8m) 2m ago 20m 507M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:34:33.118 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (7m) 9s ago 18m 451M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (7m) 2m ago 20m 50.1M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (6m) 9s ago 18m 44.9M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (12m) 2m ago 12m 274M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (12m) 9s ago 12m 87.1M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (9m) 2m ago 18m 17.1M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (8m) 9s ago 17m 18.0M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (5m) 2m ago 17m 334M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (4m) 2m ago 16m 289M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (3m) 2m ago 16m 235M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (2m) 2m ago 15m 11.8M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (2m) 9s ago 15m 352M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (91s) 9s ago 15m 310M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (54s) 9s ago 14m 288M 2704M 19.0.0-1252-g44458cce 79957b372307 653dd44c3000 2024-02-07T12:34:33.119 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (13s) 9s ago 14m 12.6M 2704M 19.0.0-1252-g44458cce 79957b372307 0445d4324ff0 2024-02-07T12:34:33.120 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (7m) 2m ago 17m 68.3M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:34:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:34:33.524 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:34:33.524 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-02-07T12:34:33.524 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:34:33.524 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:34:33.524 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-02-07T12:34:33.524 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 12 2024-02-07T12:34:33.524 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:34:33.524 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:34:33.882 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:34:33.882 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:34:33.882 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:34:33.882 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:34:33.882 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:34:33.882 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:34:33.882 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:34:33.883 INFO:teuthology.orchestra.run.smithi174.stdout: "osd", 2024-02-07T12:34:33.883 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:34:33.883 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:34:33.883 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "14/23 daemons upgraded", 2024-02-07T12:34:33.883 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading mds daemons", 2024-02-07T12:34:33.883 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:34:33.883 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:34:34.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:33 smithi174 ceph-mon[66222]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:34.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:33 smithi174 ceph-mon[66222]: osdmap e153: 8 total, 8 up, 8 in 2024-02-07T12:34:34.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:33 smithi174 ceph-mon[66222]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:34.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:33 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/2329434602' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:34.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:33 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 84/441 objects degraded (19.048%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:34.250 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:33 smithi174 ceph-mon[66222]: osdmap e154: 8 total, 8 up, 8 in 2024-02-07T12:34:34.276 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:33 smithi184 ceph-mon[53060]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:34.276 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:33 smithi184 ceph-mon[53060]: osdmap e153: 8 total, 8 up, 8 in 2024-02-07T12:34:34.276 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:33 smithi184 ceph-mon[53060]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:34.276 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:33 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/2329434602' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:34.276 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:33 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 84/441 objects degraded (19.048%), 23 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:34.276 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:33 smithi184 ceph-mon[53060]: osdmap e154: 8 total, 8 up, 8 in 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_WARN 1 filesystem is degraded; insufficient standby MDS daemons available; Degraded data redundancy: 84/441 objects degraded (19.048%), 23 pgs degraded 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout: fs foofs is degraded 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout: have 0; want 1 more 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 84/441 objects degraded (19.048%), 23 pgs degraded 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-02-07T12:34:34.337 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.7 is activating+degraded, acting [3,7,0] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.8 is activating+degraded, acting [3,1,7] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-02-07T12:34:34.338 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-02-07T12:34:34.339 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-02-07T12:34:34.339 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-02-07T12:34:34.339 INFO:teuthology.orchestra.run.smithi174.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-02-07T12:34:34.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[66218]: 2024-02-07T12:34:34.247+0000 7f6a337fb700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:34:34.546 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:34:34.271+0000 7f5aa148a700 -1 bdev(0x56376d160000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-02-07T12:34:34.547 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:34:34.275+0000 7f5aa0488700 -1 bdev(0x56376d160000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 15 2024-02-07T12:34:34.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:34:34.273+0000 7f9d8c767700 -1 bdev(0x55c5441a0000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-02-07T12:34:35.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:35.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: pgmap v301: 97 pgs: 2 activating+degraded, 1 remapped+peering, 7 peering, 5 active+undersized, 17 active+undersized+degraded, 65 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s wr, 4 op/s; 76/441 objects degraded (17.234%); 0 B/s, 6 objects/s recovering 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: osdmap e155: 8 total, 8 up, 8 in 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: Standby daemon mds.foofs.smithi184.alpixp assigned to filesystem foofs as rank 0 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: fsmap foofs:1/1 {0=foofs.smithi184.alpixp=up:replay} 2024-02-07T12:34:35.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:34 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3795373325' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:34:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:34:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: pgmap v301: 97 pgs: 2 activating+degraded, 1 remapped+peering, 7 peering, 5 active+undersized, 17 active+undersized+degraded, 65 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s wr, 4 op/s; 76/441 objects degraded (17.234%); 0 B/s, 6 objects/s recovering 2024-02-07T12:34:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-02-07T12:34:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:34:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: osdmap e155: 8 total, 8 up, 8 in 2024-02-07T12:34:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: Standby daemon mds.foofs.smithi184.alpixp assigned to filesystem foofs as rank 0 2024-02-07T12:34:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-02-07T12:34:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-07T12:34:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-02-07T12:34:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: fsmap foofs:1/1 {0=foofs.smithi184.alpixp=up:replay} 2024-02-07T12:34:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:34 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3795373325' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:34:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:36 smithi184 ceph-mon[53060]: mds.? [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] up:reconnect 2024-02-07T12:34:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:36 smithi184 ceph-mon[53060]: fsmap foofs:1/1 {0=foofs.smithi184.alpixp=up:reconnect} 2024-02-07T12:34:36.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:34:36.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:36.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:36.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:36 smithi174 ceph-mon[66222]: mds.? [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] up:reconnect 2024-02-07T12:34:36.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:36 smithi174 ceph-mon[66222]: fsmap foofs:1/1 {0=foofs.smithi184.alpixp=up:reconnect} 2024-02-07T12:34:36.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:34:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:37 smithi184 ceph-mon[53060]: reconnect by client.24327 172.21.15.184:0/782459842 after 0 2024-02-07T12:34:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:37 smithi184 ceph-mon[53060]: reconnect by client.14564 172.21.15.174:0/1930332664 after 0 2024-02-07T12:34:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:37 smithi184 ceph-mon[53060]: pgmap v303: 97 pgs: 2 activating+degraded, 1 remapped+peering, 7 peering, 87 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 11 KiB/s wr, 7 op/s; 40/441 objects degraded (9.070%); 0 B/s, 55 objects/s recovering 2024-02-07T12:34:37.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:37 smithi184 ceph-mon[53060]: mds.? [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] up:rejoin 2024-02-07T12:34:37.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:37 smithi184 ceph-mon[53060]: fsmap foofs:1/1 {0=foofs.smithi184.alpixp=up:rejoin} 2024-02-07T12:34:37.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:37 smithi184 ceph-mon[53060]: daemon mds.foofs.smithi184.alpixp is now active in filesystem foofs as rank 0 2024-02-07T12:34:37.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:37 smithi174 ceph-mon[66222]: reconnect by client.24327 172.21.15.184:0/782459842 after 0 2024-02-07T12:34:37.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:37 smithi174 ceph-mon[66222]: reconnect by client.14564 172.21.15.174:0/1930332664 after 0 2024-02-07T12:34:37.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:37 smithi174 ceph-mon[66222]: pgmap v303: 97 pgs: 2 activating+degraded, 1 remapped+peering, 7 peering, 87 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 11 KiB/s wr, 7 op/s; 40/441 objects degraded (9.070%); 0 B/s, 55 objects/s recovering 2024-02-07T12:34:37.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:37 smithi174 ceph-mon[66222]: mds.? [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] up:rejoin 2024-02-07T12:34:37.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:37 smithi174 ceph-mon[66222]: fsmap foofs:1/1 {0=foofs.smithi184.alpixp=up:rejoin} 2024-02-07T12:34:37.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:37 smithi174 ceph-mon[66222]: daemon mds.foofs.smithi184.alpixp is now active in filesystem foofs as rank 0 2024-02-07T12:34:38.375 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:38 smithi184 ceph-mon[53060]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-02-07T12:34:38.375 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:38 smithi184 ceph-mon[53060]: mds.? [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] up:active 2024-02-07T12:34:38.375 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:38 smithi184 ceph-mon[53060]: fsmap foofs:1 {0=foofs.smithi184.alpixp=up:active} 2024-02-07T12:34:38.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:38 smithi174 ceph-mon[66222]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-02-07T12:34:38.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:38 smithi174 ceph-mon[66222]: mds.? [v2:172.21.15.184:6832/2296472688,v1:172.21.15.184:6833/2296472688] up:active 2024-02-07T12:34:38.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:38 smithi174 ceph-mon[66222]: fsmap foofs:1 {0=foofs.smithi184.alpixp=up:active} 2024-02-07T12:34:39.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:39 smithi184 ceph-mon[53060]: pgmap v304: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 170 B/s wr, 10 op/s; 504 KiB/s, 45 objects/s recovering 2024-02-07T12:34:39.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:39 smithi184 ceph-mon[53060]: Health check update: Degraded data redundancy: 40/441 objects degraded (9.070%), 2 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:39.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:39 smithi184 ceph-mon[53060]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/441 objects degraded (9.070%), 2 pgs degraded) 2024-02-07T12:34:39.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:39 smithi174 ceph-mon[66222]: pgmap v304: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 170 B/s wr, 10 op/s; 504 KiB/s, 45 objects/s recovering 2024-02-07T12:34:39.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:39 smithi174 ceph-mon[66222]: Health check update: Degraded data redundancy: 40/441 objects degraded (9.070%), 2 pgs degraded (PG_DEGRADED) 2024-02-07T12:34:39.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:39 smithi174 ceph-mon[66222]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/441 objects degraded (9.070%), 2 pgs degraded) 2024-02-07T12:34:41.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:41 smithi174 ceph-mon[66222]: pgmap v305: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 143 B/s wr, 12 op/s; 424 KiB/s, 38 objects/s recovering 2024-02-07T12:34:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:41 smithi184 ceph-mon[53060]: pgmap v305: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 143 B/s wr, 12 op/s; 424 KiB/s, 38 objects/s recovering 2024-02-07T12:34:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:43 smithi184 ceph-mon[53060]: pgmap v306: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 10 KiB/s wr, 18 op/s; 364 KiB/s, 32 objects/s recovering 2024-02-07T12:34:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:43 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:43 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:43 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:43 smithi174 ceph-mon[66222]: pgmap v306: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 10 KiB/s wr, 18 op/s; 364 KiB/s, 32 objects/s recovering 2024-02-07T12:34:43.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:43 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:43.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:43 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:43.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:43 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:44.391 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:44 smithi174 ceph-mon[66222]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-02-07T12:34:44.391 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:44 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:34:44.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:44 smithi174 ceph-mon[66222]: mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525] up:boot 2024-02-07T12:34:44.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:44 smithi174 ceph-mon[66222]: fsmap foofs:1 {0=foofs.smithi184.alpixp=up:active} 1 up:standby 2024-02-07T12:34:44.392 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:44 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:34:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:44 smithi184 ceph-mon[53060]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-02-07T12:34:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:44 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:34:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:44 smithi184 ceph-mon[53060]: mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525] up:boot 2024-02-07T12:34:44.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:44 smithi184 ceph-mon[53060]: fsmap foofs:1 {0=foofs.smithi184.alpixp=up:active} 1 up:standby 2024-02-07T12:34:44.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:44 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:34:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:45 smithi184 ceph-mon[53060]: pgmap v307: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.6 KiB/s wr, 15 op/s; 302 KiB/s, 24 objects/s recovering 2024-02-07T12:34:45.470 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:45 smithi174 ceph-mon[66222]: pgmap v307: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.6 KiB/s wr, 15 op/s; 302 KiB/s, 24 objects/s recovering 2024-02-07T12:34:46.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:46 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:46.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:46 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:47.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:47 smithi174 ceph-mon[66222]: pgmap v308: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 KiB/s wr, 12 op/s; 255 KiB/s, 16 objects/s recovering 2024-02-07T12:34:47.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:47 smithi184 ceph-mon[53060]: pgmap v308: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 KiB/s wr, 12 op/s; 255 KiB/s, 16 objects/s recovering 2024-02-07T12:34:49.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:49 smithi174 ceph-mon[66222]: pgmap v309: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s; 252 KiB/s, 3 objects/s recovering 2024-02-07T12:34:49.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:49.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:49 smithi184 ceph-mon[53060]: pgmap v309: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s; 252 KiB/s, 3 objects/s recovering 2024-02-07T12:34:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:50.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:50.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:50.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:50.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:50.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:50.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:34:51.643 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:51 smithi174 ceph-mon[66222]: pgmap v310: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.1 KiB/s wr, 7 op/s 2024-02-07T12:34:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:51 smithi184 ceph-mon[53060]: pgmap v310: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.1 KiB/s wr, 7 op/s 2024-02-07T12:34:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:53 smithi184 ceph-mon[53060]: pgmap v311: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-02-07T12:34:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:53.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:53.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:53.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:53 smithi174 ceph-mon[66222]: pgmap v311: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-02-07T12:34:53.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:53.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:53.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:34:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:54.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:54 smithi184 ceph-mon[53060]: Detected new or changed devices on smithi174 2024-02-07T12:34:54.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:54.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:54.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:54.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:54.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:54.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi184.alpixp"]}]: dispatch 2024-02-07T12:34:54.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:54 smithi184 ceph-mon[53060]: Upgrade: It appears safe to stop mds.foofs.smithi184.alpixp 2024-02-07T12:34:54.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:54 smithi174 ceph-mon[66222]: Detected new or changed devices on smithi174 2024-02-07T12:34:54.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:34:54.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:54.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:54.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:54.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:34:54.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi184.alpixp"]}]: dispatch 2024-02-07T12:34:54.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:54 smithi174 ceph-mon[66222]: Upgrade: It appears safe to stop mds.foofs.smithi184.alpixp 2024-02-07T12:34:55.674 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:55 smithi184 ceph-mon[53060]: pgmap v312: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-07T12:34:55.675 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:55 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:55.675 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:55 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi184.alpixp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:34:55.675 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:55 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:55.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:55 smithi174 ceph-mon[66222]: pgmap v312: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-07T12:34:55.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:55 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:34:55.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:55 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi184.alpixp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-02-07T12:34:55.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:55 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:34:56.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:56 smithi174 ceph-mon[66222]: Upgrade: Updating mds.foofs.smithi184.alpixp 2024-02-07T12:34:56.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:56 smithi174 ceph-mon[66222]: Deploying daemon mds.foofs.smithi184.alpixp on smithi184 2024-02-07T12:34:56.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:56 smithi184 ceph-mon[53060]: Upgrade: Updating mds.foofs.smithi184.alpixp 2024-02-07T12:34:56.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:56 smithi184 ceph-mon[53060]: Deploying daemon mds.foofs.smithi184.alpixp on smithi184 2024-02-07T12:34:57.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:56 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[66218]: 2024-02-07T12:34:56.997+0000 7f6a337fb700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:34:57.297 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:34:57.013+0000 7f5aa148a700 -1 bdev(0x56376d160000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 3 2024-02-07T12:34:57.297 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:34:57.018+0000 7f5aa0c89700 -1 bdev(0x56376d160000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 12 2024-02-07T12:34:57.382 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:34:57.019+0000 7f9d8875f700 -1 bdev(0x55c5441a0000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 6 2024-02-07T12:34:57.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: pgmap v313: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-07T12:34:57.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-02-07T12:34:57.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:34:57.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: osdmap e156: 8 total, 8 up, 8 in 2024-02-07T12:34:57.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: Standby daemon mds.foofs.smithi174.vkrzkp assigned to filesystem foofs as rank 0 2024-02-07T12:34:57.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-02-07T12:34:57.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-07T12:34:57.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-02-07T12:34:57.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:57 smithi184 ceph-mon[53060]: fsmap foofs:1/1 {0=foofs.smithi174.vkrzkp=up:replay} 2024-02-07T12:34:57.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: pgmap v313: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-02-07T12:34:57.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-02-07T12:34:57.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-02-07T12:34:57.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: osdmap e156: 8 total, 8 up, 8 in 2024-02-07T12:34:57.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: Standby daemon mds.foofs.smithi174.vkrzkp assigned to filesystem foofs as rank 0 2024-02-07T12:34:57.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-02-07T12:34:57.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-02-07T12:34:57.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-02-07T12:34:57.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:57 smithi174 ceph-mon[66222]: fsmap foofs:1/1 {0=foofs.smithi174.vkrzkp=up:replay} 2024-02-07T12:34:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:59 smithi184 ceph-mon[53060]: pgmap v315: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 819 KiB/s rd, 204 B/s wr, 0 op/s 2024-02-07T12:34:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:59 smithi184 ceph-mon[53060]: reconnect by client.14564 172.21.15.174:0/1930332664 after 0.000999988 2024-02-07T12:34:59.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:59 smithi184 ceph-mon[53060]: reconnect by client.24327 172.21.15.184:0/782459842 after 0.000999988 2024-02-07T12:34:59.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:59 smithi184 ceph-mon[53060]: mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525] up:reconnect 2024-02-07T12:34:59.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:34:59 smithi184 ceph-mon[53060]: fsmap foofs:1/1 {0=foofs.smithi174.vkrzkp=up:reconnect} 2024-02-07T12:34:59.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:59 smithi174 ceph-mon[66222]: pgmap v315: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 819 KiB/s rd, 204 B/s wr, 0 op/s 2024-02-07T12:34:59.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:59 smithi174 ceph-mon[66222]: reconnect by client.14564 172.21.15.174:0/1930332664 after 0.000999988 2024-02-07T12:34:59.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:59 smithi174 ceph-mon[66222]: reconnect by client.24327 172.21.15.184:0/782459842 after 0.000999988 2024-02-07T12:34:59.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:59 smithi174 ceph-mon[66222]: mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525] up:reconnect 2024-02-07T12:34:59.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:34:59 smithi174 ceph-mon[66222]: fsmap foofs:1/1 {0=foofs.smithi174.vkrzkp=up:reconnect} 2024-02-07T12:35:00.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:00 smithi184 ceph-mon[53060]: mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525] up:rejoin 2024-02-07T12:35:00.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:00 smithi184 ceph-mon[53060]: fsmap foofs:1/1 {0=foofs.smithi174.vkrzkp=up:rejoin} 2024-02-07T12:35:00.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:00 smithi184 ceph-mon[53060]: daemon mds.foofs.smithi174.vkrzkp is now active in filesystem foofs as rank 0 2024-02-07T12:35:00.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:00 smithi174 ceph-mon[66222]: mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525] up:rejoin 2024-02-07T12:35:00.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:00 smithi174 ceph-mon[66222]: fsmap foofs:1/1 {0=foofs.smithi174.vkrzkp=up:rejoin} 2024-02-07T12:35:00.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:00 smithi174 ceph-mon[66222]: daemon mds.foofs.smithi174.vkrzkp is now active in filesystem foofs as rank 0 2024-02-07T12:35:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:01 smithi184 ceph-mon[53060]: pgmap v316: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 204 B/s wr, 2 op/s 2024-02-07T12:35:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:01 smithi184 ceph-mon[53060]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-02-07T12:35:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:01 smithi184 ceph-mon[53060]: mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525] up:active 2024-02-07T12:35:01.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:01 smithi184 ceph-mon[53060]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 2024-02-07T12:35:01.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:01 smithi174 ceph-mon[66222]: pgmap v316: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 204 B/s wr, 2 op/s 2024-02-07T12:35:01.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:01 smithi174 ceph-mon[66222]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-02-07T12:35:01.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:01 smithi174 ceph-mon[66222]: mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525] up:active 2024-02-07T12:35:01.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:01 smithi174 ceph-mon[66222]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 2024-02-07T12:35:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:03 smithi184 ceph-mon[53060]: pgmap v317: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 204 B/s wr, 10 op/s 2024-02-07T12:35:03.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:03 smithi174 ceph-mon[66222]: pgmap v317: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 204 B/s wr, 10 op/s 2024-02-07T12:35:04.695 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:35:05.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:04 smithi174 ceph-mon[66222]: pgmap v318: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 204 B/s wr, 10 op/s 2024-02-07T12:35:05.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:04 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:05.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:04 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:05.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:04 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:04 smithi184 ceph-mon[53060]: pgmap v318: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 204 B/s wr, 10 op/s 2024-02-07T12:35:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:04 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:04 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:04 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:05.352 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:35:05.352 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (10m) 17s ago 19m 22.9M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:35:05.352 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (6m) 17s ago 19m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:35:05.352 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (6m) 41s ago 18m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:35:05.352 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (9m) 17s ago 19m 52.0M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:35:05.352 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (23s) 17s ago 13m 11.4M - 19.0.0-1252-g44458cce 79957b372307 1a9cc12739f4 2024-02-07T12:35:05.352 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 starting - - - - 2024-02-07T12:35:05.352 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (9m) 17s ago 20m 512M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (8m) 41s ago 18m 451M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (7m) 17s ago 20m 60.2M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (7m) 41s ago 18m 44.9M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (13m) 17s ago 13m 287M - 4.0 e1d6a67b021e 0f85b0adbb12 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (13m) 41s ago 12m 87.1M - 4.0 e1d6a67b021e 4dabf6053b01 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (9m) 17s ago 19m 18.9M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (9m) 41s ago 18m 18.0M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (5m) 17s ago 17m 395M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (5m) 17s ago 17m 371M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (4m) 17s ago 16m 367M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (3m) 17s ago 16m 402M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (2m) 41s ago 16m 352M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (2m) 41s ago 15m 310M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (86s) 41s ago 15m 288M 2704M 19.0.0-1252-g44458cce 79957b372307 653dd44c3000 2024-02-07T12:35:05.353 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (45s) 41s ago 14m 12.6M 2704M 19.0.0-1252-g44458cce 79957b372307 0445d4324ff0 2024-02-07T12:35:05.354 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (8m) 17s ago 18m 71.0M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:35:05.750 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:35:05.750 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:35:05.750 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:35:05.750 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 14 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:35:05.751 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:35:06.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:06.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-02-07T12:35:06.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: Cluster is now healthy 2024-02-07T12:35:06.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: mds.? [v2:172.21.15.184:6832/2389971607,v1:172.21.15.184:6833/2389971607] up:boot 2024-02-07T12:35:06.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:35:06.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:35:06.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:06.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:06.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:06.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:05 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1682470411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:06.088 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "osd", 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr" 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "15/23 daemons upgraded", 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading mds daemons", 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:35:06.089 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:35:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-02-07T12:35:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: Cluster is now healthy 2024-02-07T12:35:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: mds.? [v2:172.21.15.184:6832/2389971607,v1:172.21.15.184:6833/2389971607] up:boot 2024-02-07T12:35:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: fsmap foofs:1 {0=foofs.smithi174.vkrzkp=up:active} 1 up:standby 2024-02-07T12:35:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:35:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:06.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:05 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1682470411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:06.484 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:35:07.095 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:06 smithi184 ceph-mon[53060]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:07.095 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:06 smithi184 ceph-mon[53060]: pgmap v319: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 204 B/s wr, 11 op/s 2024-02-07T12:35:07.096 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:06 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1602471731' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:35:07.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:06 smithi174 ceph-mon[66222]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:07.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:06 smithi174 ceph-mon[66222]: pgmap v319: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 204 B/s wr, 11 op/s 2024-02-07T12:35:07.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:06 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1602471731' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:35:08.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:08 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:08.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:08 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:08.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:08 smithi184 ceph-mon[53060]: pgmap v320: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.8 KiB/s wr, 13 op/s 2024-02-07T12:35:09.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:08 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:09.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:08 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:09.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:08 smithi174 ceph-mon[66222]: pgmap v320: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.8 KiB/s wr, 13 op/s 2024-02-07T12:35:09.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:09.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:09 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:10.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:10.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:09 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:10.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:10 smithi184 ceph-mon[53060]: pgmap v321: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-02-07T12:35:11.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:10 smithi174 ceph-mon[66222]: pgmap v321: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: Detected new or changed devices on smithi184 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:13.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all mds 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.vkrzkp"}]': finished 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi184.alpixp"}]': finished 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all rgw 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all iscsi 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:12 smithi174 ceph-mon[66222]: pgmap v322: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 7.2 KiB/s wr, 10 op/s 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: Detected new or changed devices on smithi184 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all mds 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.vkrzkp"}]: dispatch 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.vkrzkp"}]': finished 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi184.alpixp"}]: dispatch 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi184.alpixp"}]': finished 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all rgw 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all rbd-mirror 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all ceph-exporter 2024-02-07T12:35:13.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:13.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all iscsi 2024-02-07T12:35:13.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:13.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:12 smithi184 ceph-mon[53060]: pgmap v322: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 7.2 KiB/s wr, 10 op/s 2024-02-07T12:35:15.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: Upgrade: Updating nfs.foo.0.0.smithi174.zwcqxx (1/2) 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: Creating key for client.nfs.foo.0.0.smithi174.zwcqxx 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: Rados config object exists: conf-nfs.foo 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: Creating key for client.nfs.foo.0.0.smithi174.zwcqxx-rgw 2024-02-07T12:35:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:35:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: Bind address in nfs.foo.0.0.smithi174.zwcqxx's ganesha conf is defaulting to empty 2024-02-07T12:35:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: Deploying daemon nfs.foo.0.0.smithi174.zwcqxx on smithi174 2024-02-07T12:35:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:14 smithi174 ceph-mon[66222]: pgmap v323: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 7.1 KiB/s wr, 4 op/s 2024-02-07T12:35:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: Upgrade: Updating nfs.foo.0.0.smithi174.zwcqxx (1/2) 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: Creating key for client.nfs.foo.0.0.smithi174.zwcqxx 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: Rados config object exists: conf-nfs.foo 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: Creating key for client.nfs.foo.0.0.smithi174.zwcqxx-rgw 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi174.zwcqxx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: Bind address in nfs.foo.0.0.smithi174.zwcqxx's ganesha conf is defaulting to empty 2024-02-07T12:35:15.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:15.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: Deploying daemon nfs.foo.0.0.smithi174.zwcqxx on smithi174 2024-02-07T12:35:15.200 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:14 smithi184 ceph-mon[53060]: pgmap v323: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 7.1 KiB/s wr, 4 op/s 2024-02-07T12:35:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:17 smithi184 ceph-mon[53060]: pgmap v324: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 7.1 KiB/s wr, 4 op/s 2024-02-07T12:35:17.492 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:17 smithi174 ceph-mon[66222]: pgmap v324: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 7.1 KiB/s wr, 4 op/s 2024-02-07T12:35:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:19 smithi184 ceph-mon[53060]: pgmap v325: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 7.5 KiB/s wr, 4 op/s 2024-02-07T12:35:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:19 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:19 smithi184 ceph-mon[53060]: from='mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525]' entity='mds.foofs.smithi174.vkrzkp' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/1930332664"}]: dispatch 2024-02-07T12:35:19.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:19 smithi174 ceph-mon[66222]: pgmap v325: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 7.5 KiB/s wr, 4 op/s 2024-02-07T12:35:19.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:19.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:19 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:19.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:19 smithi174 ceph-mon[66222]: from='mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525]' entity='mds.foofs.smithi174.vkrzkp' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/1930332664"}]: dispatch 2024-02-07T12:35:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: Evicting (and blocklisting) client session 14564 (172.21.15.174:0/1930332664) 2024-02-07T12:35:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: from='mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525]' entity='mds.foofs.smithi174.vkrzkp' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/1930332664"}]': finished 2024-02-07T12:35:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: osdmap e157: 8 total, 8 up, 8 in 2024-02-07T12:35:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-07T12:35:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-07T12:35:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-07T12:35:20.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:20.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: Evicting (and blocklisting) client session 14564 (172.21.15.174:0/1930332664) 2024-02-07T12:35:20.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: from='mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525]' entity='mds.foofs.smithi174.vkrzkp' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/1930332664"}]': finished 2024-02-07T12:35:20.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: osdmap e157: 8 total, 8 up, 8 in 2024-02-07T12:35:20.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:20.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:20.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-02-07T12:35:20.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-02-07T12:35:20.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-02-07T12:35:20.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: pgmap v327: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 614 B/s wr, 1 op/s 2024-02-07T12:35:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: Upgrade: Updating nfs.foo.1.0.smithi184.rixjkj (2/2) 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: Creating key for client.nfs.foo.1.0.smithi184.rixjkj 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: Rados config object exists: conf-nfs.foo 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: Creating key for client.nfs.foo.1.0.smithi184.rixjkj-rgw 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: Bind address in nfs.foo.1.0.smithi184.rixjkj's ganesha conf is defaulting to empty 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:21.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:21 smithi184 ceph-mon[53060]: Deploying daemon nfs.foo.1.0.smithi184.rixjkj on smithi184 2024-02-07T12:35:21.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: pgmap v327: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 614 B/s wr, 1 op/s 2024-02-07T12:35:21.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: Upgrade: Updating nfs.foo.1.0.smithi184.rixjkj (2/2) 2024-02-07T12:35:21.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: Creating key for client.nfs.foo.1.0.smithi184.rixjkj 2024-02-07T12:35:21.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-02-07T12:35:21.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-02-07T12:35:21.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-02-07T12:35:21.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: Rados config object exists: conf-nfs.foo 2024-02-07T12:35:21.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: Creating key for client.nfs.foo.1.0.smithi184.rixjkj-rgw 2024-02-07T12:35:21.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi184.rixjkj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-02-07T12:35:21.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: Bind address in nfs.foo.1.0.smithi184.rixjkj's ganesha conf is defaulting to empty 2024-02-07T12:35:21.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:21.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:21 smithi174 ceph-mon[66222]: Deploying daemon nfs.foo.1.0.smithi184.rixjkj on smithi184 2024-02-07T12:35:23.376 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:23 smithi184 ceph-mon[53060]: pgmap v328: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-02-07T12:35:23.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:23 smithi174 ceph-mon[66222]: pgmap v328: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-02-07T12:35:25.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:25.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:25.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:25.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:24 smithi174 ceph-mon[66222]: pgmap v329: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-02-07T12:35:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:25.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:25.199 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:24 smithi184 ceph-mon[53060]: pgmap v329: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-02-07T12:35:27.440 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:27 smithi184 ceph-mon[53060]: pgmap v330: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-02-07T12:35:27.440 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:27.440 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:27 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:27.440 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:27 smithi184 ceph-mon[53060]: from='mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525]' entity='mds.foofs.smithi174.vkrzkp' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.184:0/782459842"}]: dispatch 2024-02-07T12:35:27.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:27 smithi174 ceph-mon[66222]: pgmap v330: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-02-07T12:35:27.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:27.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:27 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:27.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:27 smithi174 ceph-mon[66222]: from='mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525]' entity='mds.foofs.smithi174.vkrzkp' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.184:0/782459842"}]: dispatch 2024-02-07T12:35:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:28 smithi184 ceph-mon[53060]: Evicting (and blocklisting) client session 24327 (172.21.15.184:0/782459842) 2024-02-07T12:35:28.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:28 smithi184 ceph-mon[53060]: from='mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525]' entity='mds.foofs.smithi174.vkrzkp' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.184:0/782459842"}]': finished 2024-02-07T12:35:28.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:28 smithi184 ceph-mon[53060]: osdmap e158: 8 total, 8 up, 8 in 2024-02-07T12:35:28.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:28 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:28.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:28 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:28.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:28 smithi174 ceph-mon[66222]: Evicting (and blocklisting) client session 24327 (172.21.15.184:0/782459842) 2024-02-07T12:35:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:28 smithi174 ceph-mon[66222]: from='mds.? [v2:172.21.15.174:6834/2474206525,v1:172.21.15.174:6835/2474206525]' entity='mds.foofs.smithi174.vkrzkp' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.184:0/782459842"}]': finished 2024-02-07T12:35:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:28 smithi174 ceph-mon[66222]: osdmap e158: 8 total, 8 up, 8 in 2024-02-07T12:35:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:28 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:28.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:28 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:29 smithi184 ceph-mon[53060]: pgmap v332: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-02-07T12:35:29.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:29.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:29 smithi174 ceph-mon[66222]: pgmap v332: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-02-07T12:35:29.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:30.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:30.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:31.429 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:31 smithi184 ceph-mon[53060]: pgmap v333: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-02-07T12:35:31.430 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:31.430 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:31 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:31.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:31 smithi174 ceph-mon[66222]: pgmap v333: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-02-07T12:35:31.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:31.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:31 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:32.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:32.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:32 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:32.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:32 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:32.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:32 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:33.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:33 smithi174 ceph-mon[66222]: pgmap v334: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-02-07T12:35:33.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:33 smithi184 ceph-mon[53060]: pgmap v334: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-02-07T12:35:35.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.128 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:35.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:35:35.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: pgmap v335: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-02-07T12:35:35.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:35.129 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.130 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all nfs 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi174.zwcqxx"}]: dispatch 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi174.zwcqxx"}]': finished 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi184.rixjkj"}]: dispatch 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi184.rixjkj"}]': finished 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:35:35.131 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: pgmap v335: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all nfs 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi174.zwcqxx"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi174.zwcqxx"}]': finished 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi184.rixjkj"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi184.rixjkj"}]': finished 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: Upgrade: Setting container_image for all nvmeof 2024-02-07T12:35:35.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:36 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:36.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:36 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:36.871 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:35:37.423 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:37 smithi174 ceph-mon[66222]: Upgrade: Updating node-exporter.smithi174 (1/2) 2024-02-07T12:35:37.423 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:37 smithi174 ceph-mon[66222]: Deploying daemon node-exporter.smithi174 on smithi174 2024-02-07T12:35:37.423 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:37 smithi174 ceph-mon[66222]: pgmap v336: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-02-07T12:35:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:37 smithi184 ceph-mon[53060]: Upgrade: Updating node-exporter.smithi174 (1/2) 2024-02-07T12:35:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:37 smithi184 ceph-mon[53060]: Deploying daemon node-exporter.smithi174 on smithi174 2024-02-07T12:35:37.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:37 smithi184 ceph-mon[53060]: pgmap v336: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-02-07T12:35:37.566 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:35:37.566 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (10m) 8s ago 19m 23.0M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:35:37.566 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (7m) 8s ago 19m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:35:37.566 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (6m) 10s ago 19m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:35:37.566 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (10m) 8s ago 19m 52.1M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (55s) 8s ago 13m 33.0M - 19.0.0-1252-g44458cce 79957b372307 1a9cc12739f4 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (33s) 10s ago 13m 11.4M - 19.0.0-1252-g44458cce 79957b372307 d98c5145b7bb 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (9m) 8s ago 21m 513M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (8m) 10s ago 19m 451M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (8m) 8s ago 21m 61.9M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (7m) 10s ago 19m 48.2M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (19s) 8s ago 13m 69.2M - 5.7 79957b372307 4acdf8e37503 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (14s) 10s ago 13m 17.4M - 5.7 79957b372307 d08d4ce28ebc 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (10m) 8s ago 19m 19.0M - 1.3.1 1dbe0e931976 9f16bcf95c16 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (9m) 10s ago 18m 20.0M - 1.3.1 1dbe0e931976 0c65b49e73bc 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (6m) 8s ago 18m 397M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (5m) 8s ago 17m 375M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:35:37.567 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (4m) 8s ago 17m 368M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:35:37.568 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (3m) 8s ago 17m 405M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:35:37.568 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (3m) 10s ago 16m 354M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:35:37.568 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (2m) 10s ago 16m 314M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:35:37.568 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (118s) 10s ago 15m 289M 2704M 19.0.0-1252-g44458cce 79957b372307 653dd44c3000 2024-02-07T12:35:37.568 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (78s) 10s ago 15m 287M 2704M 19.0.0-1252-g44458cce 79957b372307 0445d4324ff0 2024-02-07T12:35:37.568 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (8m) 8s ago 18m 71.0M - 2.33.4 514e6a882f6e 3c4ccdea81cd 2024-02-07T12:35:37.981 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:35:37.982 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:35:37.983 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 14 2024-02-07T12:35:37.983 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:35:37.983 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:35:38.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:38 smithi184 ceph-mon[53060]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:38.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:38 smithi184 ceph-mon[53060]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:38.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:38 smithi184 ceph-mon[53060]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:38.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:38 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1811976070' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:38.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:38 smithi174 ceph-mon[66222]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:38.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:38 smithi174 ceph-mon[66222]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:38.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:38 smithi174 ceph-mon[66222]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:38.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:38 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1811976070' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout: "mds", 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:35:38.355 INFO:teuthology.orchestra.run.smithi174.stdout: "nfs", 2024-02-07T12:35:38.356 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr", 2024-02-07T12:35:38.356 INFO:teuthology.orchestra.run.smithi174.stdout: "osd" 2024-02-07T12:35:38.356 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:35:38.356 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "18/23 daemons upgraded", 2024-02-07T12:35:38.356 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading node-exporter daemons", 2024-02-07T12:35:38.356 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:35:38.356 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:35:38.795 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:35:39.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:39 smithi174 ceph-mon[66222]: pgmap v337: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 199 B/s rd, 299 B/s wr, 0 op/s 2024-02-07T12:35:39.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:39 smithi174 ceph-mon[66222]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:39.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:39 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/528801023' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:35:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:39 smithi184 ceph-mon[53060]: pgmap v337: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 199 B/s rd, 299 B/s wr, 0 op/s 2024-02-07T12:35:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:39 smithi184 ceph-mon[53060]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:35:39.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:39 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/528801023' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:35:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:41 smithi184 ceph-mon[53060]: pgmap v338: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-07T12:35:41.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:41 smithi174 ceph-mon[66222]: pgmap v338: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-02-07T12:35:42.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:42 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:42.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:42 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:42.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:42 smithi184 ceph-mon[53060]: pgmap v339: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-02-07T12:35:43.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:42 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:43.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:42 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:43.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:42 smithi174 ceph-mon[66222]: pgmap v339: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-02-07T12:35:43.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:43 smithi184 ceph-mon[53060]: Upgrade: Updating node-exporter.smithi184 (2/2) 2024-02-07T12:35:43.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:43 smithi184 ceph-mon[53060]: Deploying daemon node-exporter.smithi184 on smithi184 2024-02-07T12:35:44.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:43 smithi174 ceph-mon[66222]: Upgrade: Updating node-exporter.smithi184 (2/2) 2024-02-07T12:35:44.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:43 smithi174 ceph-mon[66222]: Deploying daemon node-exporter.smithi184 on smithi184 2024-02-07T12:35:44.858 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:44 smithi184 ceph-mon[53060]: pgmap v340: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:45.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:44 smithi174 ceph-mon[66222]: pgmap v340: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:47.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:47 smithi184 ceph-mon[53060]: pgmap v341: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:47.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:47 smithi174 ceph-mon[66222]: pgmap v341: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:49.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:49 smithi184 ceph-mon[53060]: pgmap v342: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:35:49.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:49.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:49.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:49 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:49.463 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:49 smithi174 ceph-mon[66222]: pgmap v342: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:35:49.463 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:49.463 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:49.463 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:49 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:50.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:35:51.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:51 smithi174 ceph-mon[66222]: pgmap v343: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:51 smithi184 ceph-mon[53060]: pgmap v343: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:52.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:52.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:52.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:53 smithi184 ceph-mon[53060]: pgmap v344: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:35:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:53 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:53 smithi174 ceph-mon[66222]: pgmap v344: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:35:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:53.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:53 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:54 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:54.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:54 smithi184 ceph-mon[53060]: pgmap v345: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:55.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:55.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:54 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:55.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:54 smithi174 ceph-mon[66222]: pgmap v345: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:55.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:55 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:55.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:55 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:56.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:55 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:56.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:55 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:57.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:57 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:57.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:57 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:57.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:57 smithi174 ceph-mon[66222]: pgmap v346: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:57.361 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:57 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:57.361 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:57 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:57.361 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:57 smithi184 ceph-mon[53060]: pgmap v346: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:35:59.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: pgmap v347: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:35:59.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:59.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.949 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.950 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.950 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:35:59.950 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:35:59 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: pgmap v347: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:35:59 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:01.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:01 smithi174 ceph-mon[66222]: pgmap v348: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:01.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:01 smithi174 ceph-mon[66222]: Upgrade: Updating prometheus.smithi174 2024-02-07T12:36:01.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:01 smithi174 ceph-mon[66222]: Deploying daemon prometheus.smithi174 on smithi174 2024-02-07T12:36:01.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:01 smithi184 ceph-mon[53060]: pgmap v348: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:01.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:01 smithi184 ceph-mon[53060]: Upgrade: Updating prometheus.smithi174 2024-02-07T12:36:01.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:01 smithi184 ceph-mon[53060]: Deploying daemon prometheus.smithi174 on smithi174 2024-02-07T12:36:03.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:03 smithi184 ceph-mon[53060]: pgmap v349: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:03.961 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:03 smithi174 ceph-mon[66222]: pgmap v349: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:05.145 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:04 smithi174 ceph-mon[66222]: pgmap v350: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:04 smithi184 ceph-mon[53060]: pgmap v350: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:06.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:07.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:07.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:07.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:07.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:06 smithi174 ceph-mon[66222]: pgmap v351: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:07.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:07.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:06 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:07.409 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:07.409 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:07.409 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:07.410 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:06 smithi184 ceph-mon[53060]: pgmap v351: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:07.410 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:07.410 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:06 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:09.143 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:36:09.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:09 smithi184 ceph-mon[53060]: pgmap v352: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:09.461 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:09 smithi174 ceph-mon[66222]: pgmap v352: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:09.803 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:36:09.803 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (11m) 16s ago 20m 22.7M - 0.23.0 ba2b418f427c 041d09788d2a 2024-02-07T12:36:09.803 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (7m) 16s ago 20m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:36:09.803 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (7m) 18s ago 19m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:36:09.803 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (10m) 16s ago 20m 52.4M - 8.3.5 dad864ee21e9 691f178eda3b 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (87s) 16s ago 14m 33.3M - 19.0.0-1252-g44458cce 79957b372307 1a9cc12739f4 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (65s) 18s ago 14m 11.5M - 19.0.0-1252-g44458cce 79957b372307 d98c5145b7bb 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (10m) 16s ago 21m 514M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (9m) 18s ago 19m 452M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (9m) 16s ago 21m 62.6M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (8m) 18s ago 19m 47.4M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (51s) 16s ago 14m 69.4M - 5.7 79957b372307 4acdf8e37503 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (46s) 18s ago 14m 68.2M - 5.7 79957b372307 d08d4ce28ebc 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (28s) 16s ago 20m 3565k - 1.5.0 0da6a335fe13 852c742fbac6 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (22s) 18s ago 19m 3682k - 1.5.0 0da6a335fe13 68635e4a8363 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (6m) 16s ago 18m 398M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:36:09.804 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (6m) 16s ago 18m 373M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:36:09.805 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (5m) 16s ago 18m 367M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:36:09.805 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (4m) 16s ago 17m 405M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:36:09.805 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (3m) 18s ago 17m 355M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:36:09.805 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (3m) 18s ago 16m 315M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:36:09.805 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (2m) 18s ago 16m 289M 2704M 19.0.0-1252-g44458cce 79957b372307 653dd44c3000 2024-02-07T12:36:09.805 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (110s) 18s ago 16m 287M 2704M 19.0.0-1252-g44458cce 79957b372307 0445d4324ff0 2024-02-07T12:36:09.805 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 starting - - - - 2024-02-07T12:36:10.218 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:36:10.218 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:36:10.218 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:36:10.218 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:36:10.218 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:36:10.218 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:36:10.218 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:36:10.218 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 14 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:36:10.219 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:36:10.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:10 smithi184 ceph-mon[53060]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:10.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:10 smithi184 ceph-mon[53060]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:10.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:10 smithi174 ceph-mon[66222]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:10.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:10 smithi174 ceph-mon[66222]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:10.550 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:36:10.550 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:36:10.550 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:36:10.550 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:36:10.550 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:36:10.550 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:36:10.551 INFO:teuthology.orchestra.run.smithi174.stdout: "mds", 2024-02-07T12:36:10.551 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:36:10.551 INFO:teuthology.orchestra.run.smithi174.stdout: "nfs", 2024-02-07T12:36:10.551 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr", 2024-02-07T12:36:10.551 INFO:teuthology.orchestra.run.smithi174.stdout: "osd" 2024-02-07T12:36:10.551 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:36:10.551 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "18/23 daemons upgraded", 2024-02-07T12:36:10.551 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading prometheus daemons", 2024-02-07T12:36:10.552 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:36:10.552 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:36:11.007 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:36:11.286 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:11 smithi174 ceph-mon[66222]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:11.286 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:11 smithi174 ceph-mon[66222]: pgmap v353: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:11.287 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:11 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/2310218418' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:11.287 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:11 smithi174 ceph-mon[66222]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:11.287 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:11 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3728333764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:36:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:11 smithi184 ceph-mon[53060]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:11 smithi184 ceph-mon[53060]: pgmap v353: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:11.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:11 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/2310218418' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:11.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:11 smithi184 ceph-mon[53060]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:11.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:11 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3728333764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:36:12.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:12.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:12 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:12.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:12 smithi184 ceph-mon[53060]: pgmap v354: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:13.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:13.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:12 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:13.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:12 smithi174 ceph-mon[66222]: pgmap v354: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:13.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:13.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:13 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:14.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:14.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:13 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:14 smithi174 ceph-mon[66222]: pgmap v355: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:14 smithi184 ceph-mon[53060]: pgmap v355: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:17.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: pgmap v356: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:17 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.535 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: pgmap v356: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:17.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:17 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:19 smithi184 ceph-mon[53060]: Upgrade: Updating alertmanager.smithi174 2024-02-07T12:36:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:19 smithi184 ceph-mon[53060]: Deploying daemon alertmanager.smithi174 on smithi174 2024-02-07T12:36:19.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:19 smithi184 ceph-mon[53060]: pgmap v357: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:19.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:19 smithi174 ceph-mon[66222]: Upgrade: Updating alertmanager.smithi174 2024-02-07T12:36:19.476 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:19 smithi174 ceph-mon[66222]: Deploying daemon alertmanager.smithi174 on smithi174 2024-02-07T12:36:19.477 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:19 smithi174 ceph-mon[66222]: pgmap v357: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:20.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:20.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:21.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:21 smithi184 ceph-mon[53060]: pgmap v358: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:21.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:21 smithi174 ceph-mon[66222]: pgmap v358: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:23.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:23 smithi184 ceph-mon[53060]: pgmap v359: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:23.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:23 smithi174 ceph-mon[66222]: pgmap v359: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:24.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:24.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:24.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:24.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:24.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:24 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:24.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:24 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:25.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:25 smithi174 ceph-mon[66222]: pgmap v360: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:25.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:25 smithi184 ceph-mon[53060]: pgmap v360: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:27.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:27 smithi174 ceph-mon[66222]: pgmap v361: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:27 smithi184 ceph-mon[53060]: pgmap v361: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:29.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:29 smithi184 ceph-mon[53060]: pgmap v362: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:29.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:29.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:29 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:29.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:29 smithi174 ceph-mon[66222]: pgmap v362: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:29.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:29.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:29 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:31.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:31.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:30 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:31.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:30 smithi174 ceph-mon[66222]: pgmap v363: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:30 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:31.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:30 smithi184 ceph-mon[53060]: pgmap v363: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:33.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:33 smithi174 ceph-mon[66222]: pgmap v364: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:33.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:33 smithi184 ceph-mon[53060]: pgmap v364: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:34.698 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:34.698 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:34.698 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:34.698 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:34.698 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:34.698 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:34 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:34.700 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.701 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.702 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.702 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.702 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.702 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:34.702 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:34 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:35 smithi184 ceph-mon[53060]: pgmap v365: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:35.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:35.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:35 smithi174 ceph-mon[66222]: pgmap v365: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:35.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:36.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:36 smithi184 ceph-mon[53060]: Upgrade: Updating grafana.smithi174 2024-02-07T12:36:36.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:36 smithi184 ceph-mon[53060]: Deploying daemon grafana.smithi174 on smithi174 2024-02-07T12:36:36.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:36 smithi174 ceph-mon[66222]: Upgrade: Updating grafana.smithi174 2024-02-07T12:36:36.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:36 smithi174 ceph-mon[66222]: Deploying daemon grafana.smithi174 on smithi174 2024-02-07T12:36:37.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:37 smithi184 ceph-mon[53060]: pgmap v366: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:37.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:37 smithi174 ceph-mon[66222]: pgmap v366: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:39.529 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:39 smithi174 ceph-mon[66222]: pgmap v367: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:39.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:39 smithi184 ceph-mon[53060]: pgmap v367: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:41.390 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:41 smithi174 ceph-mon[66222]: pgmap v368: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:41.390 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:41 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:41.390 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:41 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:41.390 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:41 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:41.391 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:36:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:41 smithi184 ceph-mon[53060]: pgmap v368: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:41 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:41 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:41.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:41 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:42.079 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:36:42.079 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (19s) 13s ago 21m 15.1M - 0.25.0 c8568f914cd2 86850eb9ae99 2024-02-07T12:36:42.079 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (8m) 13s ago 20m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (8m) 50s ago 20m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 starting - - - - 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (119s) 13s ago 15m 33.5M - 19.0.0-1252-g44458cce 79957b372307 1a9cc12739f4 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (97s) 50s ago 15m 11.5M - 19.0.0-1252-g44458cce 79957b372307 d98c5145b7bb 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (10m) 13s ago 22m 516M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (9m) 50s ago 20m 452M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (9m) 13s ago 22m 63.7M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (9m) 50s ago 20m 47.4M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (83s) 13s ago 14m 69.5M - 5.7 79957b372307 4acdf8e37503 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (78s) 50s ago 14m 68.2M - 5.7 79957b372307 d08d4ce28ebc 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (60s) 13s ago 20m 8288k - 1.5.0 0da6a335fe13 852c742fbac6 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (54s) 50s ago 20m 3682k - 1.5.0 0da6a335fe13 68635e4a8363 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (7m) 13s ago 19m 398M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (6m) 13s ago 19m 375M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:36:42.080 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (5m) 13s ago 18m 368M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:36:42.081 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (5m) 13s ago 18m 405M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:36:42.081 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (4m) 50s ago 17m 355M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:36:42.081 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (3m) 50s ago 17m 315M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:36:42.081 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (3m) 50s ago 16m 289M 2704M 19.0.0-1252-g44458cce 79957b372307 653dd44c3000 2024-02-07T12:36:42.081 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (2m) 50s ago 16m 287M 2704M 19.0.0-1252-g44458cce 79957b372307 0445d4324ff0 2024-02-07T12:36:42.081 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (36s) 13s ago 19m 48.8M - 2.43.0 a07b618ecd1d 42d55840b88f 2024-02-07T12:36:42.220 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:42 smithi174 ceph-mon[66222]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:42.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:42 smithi184 ceph-mon[53060]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:36:42.494 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:36:42.495 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:36:42.495 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 14 2024-02-07T12:36:42.495 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:36:42.495 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:36:42.837 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:36:42.837 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", 2024-02-07T12:36:42.837 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": true, 2024-02-07T12:36:42.837 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "Upgrading all daemon types on all hosts", 2024-02-07T12:36:42.837 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [ 2024-02-07T12:36:42.837 INFO:teuthology.orchestra.run.smithi174.stdout: "mon", 2024-02-07T12:36:42.837 INFO:teuthology.orchestra.run.smithi174.stdout: "mds", 2024-02-07T12:36:42.837 INFO:teuthology.orchestra.run.smithi174.stdout: "crash", 2024-02-07T12:36:42.838 INFO:teuthology.orchestra.run.smithi174.stdout: "nfs", 2024-02-07T12:36:42.838 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr", 2024-02-07T12:36:42.838 INFO:teuthology.orchestra.run.smithi174.stdout: "osd" 2024-02-07T12:36:42.838 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:36:42.838 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": "18/23 daemons upgraded", 2024-02-07T12:36:42.838 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "Currently upgrading grafana daemons", 2024-02-07T12:36:42.838 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:36:42.838 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:36:43.254 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:36:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:43 smithi184 ceph-mon[53060]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:43 smithi184 ceph-mon[53060]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:43 smithi184 ceph-mon[53060]: pgmap v369: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:43.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:43 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/373812638' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:43 smithi174 ceph-mon[66222]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:43 smithi174 ceph-mon[66222]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:43 smithi174 ceph-mon[66222]: pgmap v369: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:43 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/373812638' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:44.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:44 smithi174 ceph-mon[66222]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:44.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:44 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3968545611' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:36:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:44 smithi184 ceph-mon[53060]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:36:44.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:44 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3968545611' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:36:45.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:45 smithi174 ceph-mon[66222]: pgmap v370: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:45.448 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:45 smithi184 ceph-mon[53060]: pgmap v370: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:46.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:46 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:46.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:46 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:47.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:46 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:47.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:46 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:47.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:47 smithi174 ceph-mon[66222]: pgmap v371: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:47.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:47 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:47.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:47 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:47 smithi184 ceph-mon[53060]: pgmap v371: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:47 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:47.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:47 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:49.948 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:49 smithi184 ceph-mon[53060]: pgmap v372: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:49.959 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:49 smithi174 ceph-mon[66222]: pgmap v372: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:51.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: pgmap v373: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: Upgrade: Finalizing container_image settings 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:36:51.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:36:51.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: Upgrade: Complete! 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:51.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:51.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:51.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:36:51.449 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: pgmap v373: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.450 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: Upgrade: Finalizing container_image settings 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-02-07T12:36:51.451 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-02-07T12:36:51.452 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: Upgrade: Complete! 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-02-07T12:36:51.453 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:51.454 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:36:53.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:53 smithi174 ceph-mon[66222]: pgmap v374: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:53 smithi184 ceph-mon[53060]: pgmap v374: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:54 smithi184 ceph-mon[53060]: pgmap v375: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:55.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:54 smithi174 ceph-mon[66222]: pgmap v375: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:57.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:57 smithi174 ceph-mon[66222]: pgmap v376: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:57.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:57 smithi184 ceph-mon[53060]: pgmap v376: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:36:59.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:36:59 smithi174 ceph-mon[66222]: pgmap v377: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:36:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:36:59 smithi184 ceph-mon[53060]: pgmap v377: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:01.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:01 smithi174 ceph-mon[66222]: pgmap v378: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:01 smithi184 ceph-mon[53060]: pgmap v378: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:03.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:03 smithi174 ceph-mon[66222]: pgmap v379: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:03 smithi184 ceph-mon[53060]: pgmap v379: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:04 smithi184 ceph-mon[53060]: pgmap v380: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:05.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:04 smithi174 ceph-mon[66222]: pgmap v380: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:06.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:07.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:06 smithi184 ceph-mon[53060]: pgmap v381: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:07.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:06 smithi174 ceph-mon[66222]: pgmap v381: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:09.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:09 smithi174 ceph-mon[66222]: pgmap v382: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:09 smithi184 ceph-mon[53060]: pgmap v382: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:11.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:11 smithi174 ceph-mon[66222]: pgmap v383: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:11 smithi184 ceph-mon[53060]: pgmap v383: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:13.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:13 smithi174 ceph-mon[66222]: pgmap v384: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:13 smithi184 ceph-mon[53060]: pgmap v384: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:14.311 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:37:14.923 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:14 smithi174 ceph-mon[66222]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:37:14.923 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:14 smithi174 ceph-mon[66222]: pgmap v385: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:14 smithi184 ceph-mon[53060]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:37:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:14 smithi184 ceph-mon[53060]: pgmap v385: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (53s) 30s ago 21m 15.1M - 0.25.0 c8568f914cd2 86850eb9ae99 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (8m) 30s ago 21m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (8m) 85s ago 20m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (36s) 30s ago 21m 51.1M - 9.4.12 a0ea5e3c46d9 2b870bccdf3a 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (2m) 30s ago 15m 33.7M - 19.0.0-1252-g44458cce 79957b372307 1a9cc12739f4 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (2m) 85s ago 15m 11.5M - 19.0.0-1252-g44458cce 79957b372307 d98c5145b7bb 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (11m) 30s ago 22m 517M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:37:16.414 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (10m) 85s ago 20m 452M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (10m) 30s ago 22m 64.3M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (9m) 85s ago 20m 47.4M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (118s) 30s ago 15m 69.5M - 5.7 79957b372307 4acdf8e37503 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (112s) 85s ago 15m 68.2M - 5.7 79957b372307 d08d4ce28ebc 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (95s) 30s ago 21m 8460k - 1.5.0 0da6a335fe13 852c742fbac6 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (88s) 85s ago 20m 3682k - 1.5.0 0da6a335fe13 68635e4a8363 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (8m) 30s ago 20m 399M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (7m) 30s ago 19m 375M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (6m) 30s ago 19m 368M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (5m) 30s ago 18m 405M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (4m) 85s ago 18m 355M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (4m) 85s ago 17m 315M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (3m) 85s ago 17m 289M 2704M 19.0.0-1252-g44458cce 79957b372307 653dd44c3000 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (2m) 85s ago 17m 287M 2704M 19.0.0-1252-g44458cce 79957b372307 0445d4324ff0 2024-02-07T12:37:16.415 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (70s) 30s ago 20m 49.0M - 2.43.0 a07b618ecd1d 42d55840b88f 2024-02-07T12:37:17.016 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:37:17.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:17 smithi174 ceph-mon[66222]: pgmap v386: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:17.545 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:17 smithi174 ceph-mon[66222]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:37:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:17 smithi184 ceph-mon[53060]: pgmap v386: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:17 smithi184 ceph-mon[53060]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:37:19.063 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:37:19.064 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:37:19.064 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:37:19.064 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:37:19.064 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:37:19.064 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:37:19.064 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 14 2024-02-07T12:37:19.064 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:37:19.064 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:37:19.415 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:19 smithi174 ceph-mon[66222]: pgmap v387: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:19.415 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:19 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/916158351' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:37:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:19 smithi184 ceph-mon[53060]: pgmap v387: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:19 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/916158351' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:37:19.757 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-02-07T12:37:20.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:20.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:21.285 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:21 smithi174 ceph-mon[66222]: pgmap v388: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:21.517 INFO:teuthology.orchestra.run.smithi174.stdout:wait for servicemap items w/ changing names to refresh 2024-02-07T12:37:21.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:21 smithi184 ceph-mon[53060]: pgmap v388: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:22.179 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'sleep 60' 2024-02-07T12:37:23.516 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:23 smithi174 ceph-mon[66222]: pgmap v389: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:23 smithi184 ceph-mon[53060]: pgmap v389: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:24 smithi184 ceph-mon[53060]: pgmap v390: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:25.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:24 smithi174 ceph-mon[66222]: pgmap v390: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:27.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:27 smithi174 ceph-mon[66222]: pgmap v391: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:27 smithi184 ceph-mon[53060]: pgmap v391: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:29.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:29 smithi174 ceph-mon[66222]: pgmap v392: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:29.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:29 smithi184 ceph-mon[53060]: pgmap v392: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:31.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:31 smithi174 ceph-mon[66222]: pgmap v393: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:31.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:31 smithi184 ceph-mon[53060]: pgmap v393: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:33.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:33 smithi174 ceph-mon[66222]: pgmap v394: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:33 smithi184 ceph-mon[53060]: pgmap v394: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:35.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:34 smithi184 ceph-mon[53060]: pgmap v395: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:35.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:34 smithi174 ceph-mon[66222]: pgmap v395: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:36.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:36.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:37.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:36 smithi184 ceph-mon[53060]: pgmap v396: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:37.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:36 smithi174 ceph-mon[66222]: pgmap v396: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:39.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:39 smithi174 ceph-mon[66222]: pgmap v397: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:39.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:39 smithi184 ceph-mon[53060]: pgmap v397: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:41.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:41 smithi174 ceph-mon[66222]: pgmap v398: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:41.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:41 smithi184 ceph-mon[53060]: pgmap v398: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:43.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:43 smithi174 ceph-mon[66222]: pgmap v399: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:43 smithi184 ceph-mon[53060]: pgmap v399: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:45.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:44 smithi184 ceph-mon[53060]: pgmap v400: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:45.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:44 smithi174 ceph-mon[66222]: pgmap v400: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:47.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:47 smithi174 ceph-mon[66222]: pgmap v401: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:47.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:47 smithi184 ceph-mon[53060]: pgmap v401: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:49.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:49 smithi174 ceph-mon[66222]: pgmap v402: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:49 smithi184 ceph-mon[53060]: pgmap v402: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:50.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:50.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:37:51.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:51 smithi174 ceph-mon[66222]: pgmap v403: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:51.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:37:51.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:37:51.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:37:51.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:37:51.547 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:37:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:51 smithi184 ceph-mon[53060]: pgmap v403: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:37:51.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:37:51.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:37:51.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:37:51.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:37:53.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:53 smithi174 ceph-mon[66222]: pgmap v404: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:53.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:53 smithi184 ceph-mon[53060]: pgmap v404: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:54 smithi184 ceph-mon[53060]: pgmap v405: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:55.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:54 smithi174 ceph-mon[66222]: pgmap v405: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:57.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:57 smithi174 ceph-mon[66222]: pgmap v406: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:57.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:57 smithi184 ceph-mon[53060]: pgmap v406: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:37:59.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:37:59 smithi174 ceph-mon[66222]: pgmap v407: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:37:59.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:37:59 smithi184 ceph-mon[53060]: pgmap v407: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:01.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:01 smithi174 ceph-mon[66222]: pgmap v408: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:01.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:01 smithi184 ceph-mon[53060]: pgmap v408: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:03.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:03 smithi174 ceph-mon[66222]: pgmap v409: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:03.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:03 smithi184 ceph-mon[53060]: pgmap v409: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:05.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:04 smithi184 ceph-mon[53060]: pgmap v410: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:05.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:04 smithi174 ceph-mon[66222]: pgmap v410: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:06.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:05 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:06.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:05 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:07.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:06 smithi184 ceph-mon[53060]: pgmap v411: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:07.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:06 smithi174 ceph-mon[66222]: pgmap v411: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:09.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:09 smithi174 ceph-mon[66222]: pgmap v412: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:09.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:09 smithi184 ceph-mon[53060]: pgmap v412: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:11.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:11 smithi174 ceph-mon[66222]: pgmap v413: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:11.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:11 smithi184 ceph-mon[53060]: pgmap v413: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:13.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:13 smithi174 ceph-mon[66222]: pgmap v414: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:13.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:13 smithi184 ceph-mon[53060]: pgmap v414: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:15.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:14 smithi184 ceph-mon[53060]: pgmap v415: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:15.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:14 smithi174 ceph-mon[66222]: pgmap v415: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:17.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:17 smithi174 ceph-mon[66222]: pgmap v416: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:17.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:17 smithi184 ceph-mon[53060]: pgmap v416: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:19.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:19 smithi174 ceph-mon[66222]: pgmap v417: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:19.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:19 smithi184 ceph-mon[53060]: pgmap v417: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:20.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:20 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:20.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:20 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:21.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:21 smithi174 ceph-mon[66222]: pgmap v418: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:21.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:21 smithi184 ceph-mon[53060]: pgmap v418: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:23.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:23 smithi174 ceph-mon[66222]: pgmap v419: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:23.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:23 smithi184 ceph-mon[53060]: pgmap v419: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:24.450 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch ps' 2024-02-07T12:38:25.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:24 smithi174 ceph-mon[66222]: pgmap v420: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:25.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:24 smithi184 ceph-mon[53060]: pgmap v420: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:26.427 INFO:teuthology.orchestra.run.smithi174.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-02-07T12:38:26.427 INFO:teuthology.orchestra.run.smithi174.stdout:alertmanager.smithi174 smithi174 *:9093,9094 running (2m) 100s ago 22m 15.1M - 0.25.0 c8568f914cd2 86850eb9ae99 2024-02-07T12:38:26.427 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi174 smithi174 running (9m) 100s ago 22m 6656k - 19.0.0-1252-g44458cce 79957b372307 77a946a66699 2024-02-07T12:38:26.427 INFO:teuthology.orchestra.run.smithi174.stdout:crash.smithi184 smithi184 running (9m) 2m ago 22m 6660k - 19.0.0-1252-g44458cce 79957b372307 60ae73cf6615 2024-02-07T12:38:26.427 INFO:teuthology.orchestra.run.smithi174.stdout:grafana.smithi174 smithi174 *:3000 running (106s) 100s ago 22m 51.1M - 9.4.12 a0ea5e3c46d9 2b870bccdf3a 2024-02-07T12:38:26.427 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi174.vkrzkp smithi174 running (3m) 100s ago 16m 33.7M - 19.0.0-1252-g44458cce 79957b372307 1a9cc12739f4 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:mds.foofs.smithi184.alpixp smithi184 running (3m) 2m ago 16m 11.5M - 19.0.0-1252-g44458cce 79957b372307 d98c5145b7bb 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi174.whsozo smithi174 *:8443,9283,8765 running (12m) 100s ago 23m 517M - 19.0.0-1252-g44458cce 79957b372307 edc3d6c0e08c 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:mgr.smithi184.prgadw smithi184 *:8443,9283,8765 running (11m) 2m ago 21m 452M - 19.0.0-1252-g44458cce 79957b372307 7f56655d341c 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi174 smithi174 running (11m) 100s ago 24m 64.3M 2048M 19.0.0-1252-g44458cce 79957b372307 245621856c3f 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:mon.smithi184 smithi184 running (10m) 2m ago 21m 47.4M 2048M 19.0.0-1252-g44458cce 79957b372307 40219d987787 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.0.0.smithi174.zwcqxx smithi174 *:2049 running (3m) 100s ago 16m 69.5M - 5.7 79957b372307 4acdf8e37503 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:nfs.foo.1.0.smithi184.rixjkj smithi184 *:2049 running (3m) 2m ago 16m 68.2M - 5.7 79957b372307 d08d4ce28ebc 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi174 smithi174 *:9100 running (2m) 100s ago 22m 8460k - 1.5.0 0da6a335fe13 852c742fbac6 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:node-exporter.smithi184 smithi184 *:9100 running (2m) 2m ago 21m 3682k - 1.5.0 0da6a335fe13 68635e4a8363 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:osd.0 smithi174 running (9m) 100s ago 21m 399M 1936M 19.0.0-1252-g44458cce 79957b372307 aa986beb783d 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:osd.1 smithi174 running (8m) 100s ago 20m 375M 1936M 19.0.0-1252-g44458cce 79957b372307 6e9164f04adb 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:osd.2 smithi174 running (7m) 100s ago 20m 368M 1936M 19.0.0-1252-g44458cce 79957b372307 5dcb68e59527 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:osd.3 smithi174 running (6m) 100s ago 19m 405M 1936M 19.0.0-1252-g44458cce 79957b372307 ea1ed26d4476 2024-02-07T12:38:26.428 INFO:teuthology.orchestra.run.smithi174.stdout:osd.4 smithi184 running (6m) 2m ago 19m 355M 2704M 19.0.0-1252-g44458cce 79957b372307 2c6c37d4be2a 2024-02-07T12:38:26.429 INFO:teuthology.orchestra.run.smithi174.stdout:osd.5 smithi184 running (5m) 2m ago 19m 315M 2704M 19.0.0-1252-g44458cce 79957b372307 8b5693e2bfd0 2024-02-07T12:38:26.429 INFO:teuthology.orchestra.run.smithi174.stdout:osd.6 smithi184 running (4m) 2m ago 18m 289M 2704M 19.0.0-1252-g44458cce 79957b372307 653dd44c3000 2024-02-07T12:38:26.429 INFO:teuthology.orchestra.run.smithi174.stdout:osd.7 smithi184 running (4m) 2m ago 18m 287M 2704M 19.0.0-1252-g44458cce 79957b372307 0445d4324ff0 2024-02-07T12:38:26.429 INFO:teuthology.orchestra.run.smithi174.stdout:prometheus.smithi174 smithi174 *:9095 running (2m) 100s ago 21m 49.0M - 2.43.0 a07b618ecd1d 42d55840b88f 2024-02-07T12:38:27.105 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph orch upgrade status' 2024-02-07T12:38:27.415 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:27 smithi174 ceph-mon[66222]: pgmap v421: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:27.416 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:27 smithi174 ceph-mon[66222]: from='client.34640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:38:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:27 smithi184 ceph-mon[53060]: pgmap v421: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:27.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:27 smithi184 ceph-mon[53060]: from='client.34640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:38:29.109 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:38:29.109 INFO:teuthology.orchestra.run.smithi174.stdout: "target_image": null, 2024-02-07T12:38:29.109 INFO:teuthology.orchestra.run.smithi174.stdout: "in_progress": false, 2024-02-07T12:38:29.109 INFO:teuthology.orchestra.run.smithi174.stdout: "which": "", 2024-02-07T12:38:29.109 INFO:teuthology.orchestra.run.smithi174.stdout: "services_complete": [], 2024-02-07T12:38:29.109 INFO:teuthology.orchestra.run.smithi174.stdout: "progress": null, 2024-02-07T12:38:29.110 INFO:teuthology.orchestra.run.smithi174.stdout: "message": "", 2024-02-07T12:38:29.110 INFO:teuthology.orchestra.run.smithi174.stdout: "is_paused": false 2024-02-07T12:38:29.110 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:38:29.443 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:29 smithi174 ceph-mon[66222]: pgmap v422: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:29.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:29 smithi184 ceph-mon[53060]: pgmap v422: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:29.757 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph health detail' 2024-02-07T12:38:30.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:30 smithi174 ceph-mon[66222]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:38:30.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:30 smithi184 ceph-mon[53060]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:38:31.358 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:31 smithi174 ceph-mon[66222]: pgmap v423: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:31.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:31 smithi184 ceph-mon[53060]: pgmap v423: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:31.851 INFO:teuthology.orchestra.run.smithi174.stdout:HEALTH_OK 2024-02-07T12:38:32.465 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:32 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/4169648177' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:38:32.486 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions' 2024-02-07T12:38:32.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:32 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/4169648177' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-02-07T12:38:33.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:33 smithi174 ceph-mon[66222]: pgmap v424: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:33.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:33 smithi184 ceph-mon[53060]: pgmap v424: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: "mon": { 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: "mgr": { 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: "osd": { 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 8 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:38:34.528 INFO:teuthology.orchestra.run.smithi174.stdout: "mds": { 2024-02-07T12:38:34.529 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 2 2024-02-07T12:38:34.529 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:38:34.529 INFO:teuthology.orchestra.run.smithi174.stdout: "overall": { 2024-02-07T12:38:34.529 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)": 14 2024-02-07T12:38:34.529 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:38:34.529 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-02-07T12:38:35.111 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:34 smithi174 ceph-mon[66222]: pgmap v425: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:35.111 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:34 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/1417362514' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:38:35.134 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-02-07T12:38:35.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:34 smithi184 ceph-mon[53060]: pgmap v425: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:35.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:34 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/1417362514' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:38:36.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:35 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:36.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:35 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:37.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:36 smithi174 ceph-mon[66222]: pgmap v426: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:37.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:36 smithi184 ceph-mon[53060]: pgmap v426: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:37.233 INFO:teuthology.orchestra.run.smithi174.stdout:true 2024-02-07T12:38:37.814 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -e sha1=44458ccedb18241cc2e17f10cdab3d1385399261 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-02-07T12:38:38.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:37 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3613725215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:38:38.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:37 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3613725215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:38:39.155 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:38 smithi174 ceph-mon[66222]: pgmap v427: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:39.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:38 smithi184 ceph-mon[53060]: pgmap v427: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:39.735 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph version 19.0.0-1252-g44458cce (44458ccedb18241cc2e17f10cdab3d1385399261) squid (dev)" 2024-02-07T12:38:40.046 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:39 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/30557786' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:38:40.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:39 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/30557786' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-02-07T12:38:40.477 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-02-07T12:38:40.478 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-02-07T12:38:40.478 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- ceph orch ls -f json 2024-02-07T12:38:41.039 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:40 smithi174 ceph-mon[66222]: pgmap v428: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:41.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:40 smithi184 ceph-mon[53060]: pgmap v428: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:42.415 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-02-07T12:38:42.416 INFO:teuthology.orchestra.run.smithi174.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-02-07T12:15:03.733694Z", "last_refresh": "2024-02-07T12:36:45.541358Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-02-07T12:14:57.633733Z", "last_refresh": "2024-02-07T12:35:51.266150Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-02-07T12:15:00.597245Z", "last_refresh": "2024-02-07T12:36:45.541500Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-02-07T12:21:34.234588Z", "last_refresh": "2024-02-07T12:35:51.266943Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-02-07T12:14:55.497950Z", "last_refresh": "2024-02-07T12:35:51.266320Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi174:172.21.15.174=smithi174", "smithi184:172.21.15.184=smithi184"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-02-07T12:15:47.354666Z", "last_refresh": "2024-02-07T12:35:51.266463Z", "running": 2, "size": 2}}, {"events": ["2024-02-07T12:37:51.111895Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-02-07T12:21:56.323381Z", "last_refresh": "2024-02-07T12:35:51.267019Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-02-07T12:15:01.843096Z", "last_refresh": "2024-02-07T12:35:51.266562Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe", "last_refresh": "2024-02-07T12:35:51.266641Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-02-07T12:14:58.912248Z", "last_refresh": "2024-02-07T12:36:45.541641Z", "ports": [9095], "running": 1, "size": 1}}] 2024-02-07T12:38:43.024 INFO:tasks.cephadm:nfs.foo has 2/2 2024-02-07T12:38:43.025 DEBUG:teuthology.parallel:result is None 2024-02-07T12:38:43.025 INFO:teuthology.run_tasks:Running task vip.exec... 2024-02-07T12:38:43.037 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:38:43.038 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-02-07T12:38:43.066 INFO:teuthology.orchestra.run.smithi174.stderr:+ umount /mnt/foo 2024-02-07T12:38:43.078 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-07T12:38:43.087 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:38:43.087 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-02-07T12:38:43.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:43 smithi174 ceph-mon[66222]: pgmap v429: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:43.296 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:43 smithi174 ceph-mon[66222]: from='client.34664 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:38:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:43 smithi184 ceph-mon[53060]: pgmap v429: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:43.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:43 smithi184 ceph-mon[53060]: from='client.34664 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-02-07T12:38:45.169 INFO:teuthology.orchestra.run.smithi174.stdout: "foo" 2024-02-07T12:38:45.169 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:44 smithi174 ceph-mon[66222]: pgmap v430: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:45.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:44 smithi184 ceph-mon[53060]: pgmap v430: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:45.837 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'ceph nfs export ls foo --detailed' 2024-02-07T12:38:46.143 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:45 smithi174 ceph-mon[66222]: from='client.34668 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:38:46.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:45 smithi184 ceph-mon[53060]: from='client.34668 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:38:47.094 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:46 smithi174 ceph-mon[66222]: pgmap v431: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:47.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:46 smithi184 ceph-mon[53060]: pgmap v431: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:47.867 INFO:teuthology.orchestra.run.smithi174.stdout:[ 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: { 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "access_type": "RW", 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "clients": [], 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "cluster_id": "foo", 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "export_id": 1, 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "fsal": { 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "fs_name": "foofs", 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "name": "CEPH", 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "user_id": "nfs.foo.1" 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "path": "/", 2024-02-07T12:38:47.868 INFO:teuthology.orchestra.run.smithi174.stdout: "protocols": [ 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: 4 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: "pseudo": "/fake", 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: "security_label": true, 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: "squash": "none", 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: "transports": [ 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: "TCP" 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: ] 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout: } 2024-02-07T12:38:47.869 INFO:teuthology.orchestra.run.smithi174.stdout:] 2024-02-07T12:38:48.474 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'rados -p .nfs --all ls -' 2024-02-07T12:38:49.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:49 smithi174 ceph-mon[66222]: from='client.34672 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:38:49.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:49 smithi174 ceph-mon[66222]: pgmap v432: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:49.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:49 smithi184 ceph-mon[53060]: from='client.34672 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-02-07T12:38:49.699 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:49 smithi184 ceph-mon[53060]: pgmap v432: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-02-07T12:38:50.244 INFO:teuthology.orchestra.run.smithi174.stdout:foo rec-0000000000000004:nfs.foo.1 2024-02-07T12:38:50.245 INFO:teuthology.orchestra.run.smithi174.stdout:foo grace 2024-02-07T12:38:50.245 INFO:teuthology.orchestra.run.smithi174.stdout:foo rec-0000000000000004:nfs.foo.0 2024-02-07T12:38:50.245 INFO:teuthology.orchestra.run.smithi174.stdout:foo conf-nfs.foo 2024-02-07T12:38:50.245 INFO:teuthology.orchestra.run.smithi174.stdout:foo export-1 2024-02-07T12:38:50.864 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-02-07T12:38:50.874 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:38:50.875 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 -- bash -c 'set -ex 2024-02-07T12:38:50.875 DEBUG:teuthology.orchestra.run.smithi174:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-02-07T12:38:50.875 DEBUG:teuthology.orchestra.run.smithi174:> ' 2024-02-07T12:38:51.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:50 smithi174 ceph-mon[66222]: mgrmap e42: smithi174.whsozo(active, since 11m), standbys: smithi184.prgadw 2024-02-07T12:38:51.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:50 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:51.182 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:50 smithi174 ceph-mon[66222]: pgmap v433: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:50 smithi184 ceph-mon[53060]: mgrmap e42: smithi174.whsozo(active, since 11m), standbys: smithi184.prgadw 2024-02-07T12:38:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:50 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-02-07T12:38:51.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:50 smithi184 ceph-mon[53060]: pgmap v433: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-02-07T12:38:52.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:51 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:38:52.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:51 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-02-07T12:38:52.633 INFO:teuthology.orchestra.run.smithi174.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-02-07T12:38:52.834 INFO:teuthology.orchestra.run.smithi174.stderr:+ [[ 6 -gt 2 ]] 2024-02-07T12:38:53.146 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:38:53.146 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:38:53.146 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:38:53.146 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:52 smithi174 ceph-mon[66222]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:38:53.146 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:52 smithi174 ceph-mon[66222]: pgmap v434: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-07T12:38:53.146 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:52 smithi174 ceph-mon[66222]: from='client.? 172.21.15.174:0/3106761177' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-02-07T12:38:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-02-07T12:38:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-02-07T12:38:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:38:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:52 smithi184 ceph-mon[53060]: from='mgr.34102 172.21.15.174:0/1453125390' entity='mgr.smithi174.whsozo' 2024-02-07T12:38:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:52 smithi184 ceph-mon[53060]: pgmap v434: 97 pgs: 97 active+clean; 245 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-02-07T12:38:53.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:52 smithi184 ceph-mon[53060]: from='client.? 172.21.15.174:0/3106761177' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-02-07T12:38:53.478 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-02-07T12:38:53.489 INFO:tasks.cephadm:Teardown begin 2024-02-07T12:38:53.490 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:38:53.519 DEBUG:teuthology.orchestra.run.smithi184:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:38:53.554 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-02-07T12:38:53.554 DEBUG:teuthology.orchestra.run.smithi174:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-02-07T12:38:53.574 DEBUG:teuthology.orchestra.run.smithi184:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-02-07T12:38:53.611 INFO:tasks.cephadm:Stopping all daemons... 2024-02-07T12:38:53.612 INFO:tasks.cephadm.mon.smithi174:Stopping mon.smithi174... 2024-02-07T12:38:53.612 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174 2024-02-07T12:38:53.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:53 smithi174 systemd[1]: Stopping Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:38:54.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:53 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[66218]: 2024-02-07T12:38:53.921+0000 7f6a39006700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi174 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:38:54.252 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:53 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174[66218]: 2024-02-07T12:38:53.921+0000 7f6a39006700 -1 mon.smithi174@0(leader) e3 *** Got Signal Terminated *** 2024-02-07T12:38:54.252 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:53 smithi174 podman[112078]: 2024-02-07 12:38:53.942500953 +0000 UTC m=+0.143730085 container died 245621856c3fd9c4bab9556341ce5f8f875efaa56ef8340f34573692b82f8f9a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-02-07T12:38:54.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:54 smithi174 podman[112078]: 2024-02-07 12:38:54.35556547 +0000 UTC m=+0.556794609 container cleanup 245621856c3fd9c4bab9556341ce5f8f875efaa56ef8340f34573692b82f8f9a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS) 2024-02-07T12:38:54.546 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:54 smithi174 bash[112078]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174 2024-02-07T12:38:54.824 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service' 2024-02-07T12:38:54.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:54 smithi174 podman[112093]: 2024-02-07 12:38:54.582213705 +0000 UTC m=+0.638930584 container remove 245621856c3fd9c4bab9556341ce5f8f875efaa56ef8340f34573692b82f8f9a (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi174, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131) 2024-02-07T12:38:54.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:54 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service: Deactivated successfully. 2024-02-07T12:38:54.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:54 smithi174 systemd[1]: Stopped Ceph mon.smithi174 for 53d92d24-c5b2-11ee-95b6-87774f69a715. 2024-02-07T12:38:54.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Feb 07 12:38:54 smithi174 systemd[1]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi174.service: Consumed 15.311s CPU time. 2024-02-07T12:38:54.887 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:38:54.887 INFO:tasks.cephadm.mon.smithi174:Stopped mon.smithi174 2024-02-07T12:38:54.887 INFO:tasks.cephadm.mon.smithi184:Stopping mon.smithi184... 2024-02-07T12:38:54.888 DEBUG:teuthology.orchestra.run.smithi184:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi184 2024-02-07T12:38:55.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:54 smithi184 systemd[1]: Stopping Ceph mon.smithi184 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:38:55.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:55 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184[53056]: 2024-02-07T12:38:55.348+0000 7f6136b0b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi184 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:38:55.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:55 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184[53056]: 2024-02-07T12:38:55.348+0000 7f6136b0b700 -1 mon.smithi184@1(peon) e3 *** Got Signal Terminated *** 2024-02-07T12:38:55.698 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:55 smithi184 podman[78342]: 2024-02-07 12:38:55.532802127 +0000 UTC m=+0.453842599 container died 40219d987787438b07b700b3d017e3e329fbf44fe8cce7ba344b8d189e4a1451 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-02-07T12:38:56.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:55 smithi184 podman[78342]: 2024-02-07 12:38:55.99128587 +0000 UTC m=+0.912326341 container cleanup 40219d987787438b07b700b3d017e3e329fbf44fe8cce7ba344b8d189e4a1451 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-02-07T12:38:56.198 INFO:journalctl@ceph.mon.smithi184.smithi184.stdout:Feb 07 12:38:55 smithi184 bash[78342]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-mon-smithi184 2024-02-07T12:38:56.419 DEBUG:teuthology.orchestra.run.smithi184:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@mon.smithi184.service' 2024-02-07T12:38:56.468 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:38:56.468 INFO:tasks.cephadm.mon.smithi184:Stopped mon.smithi184 2024-02-07T12:38:56.469 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-02-07T12:38:56.469 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.0 2024-02-07T12:38:56.546 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:38:56 smithi174 systemd[1]: Stopping Ceph osd.0 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:38:57.046 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:38:56 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[72402]: 2024-02-07T12:38:56.759+0000 7fae9fe90700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:38:57.046 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:38:56 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[72402]: 2024-02-07T12:38:56.759+0000 7fae9fe90700 -1 osd.0 158 *** Got signal Terminated *** 2024-02-07T12:38:57.047 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:38:56 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0[72402]: 2024-02-07T12:38:56.759+0000 7fae9fe90700 -1 osd.0 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:39:02.521 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:02 smithi174 podman[112244]: 2024-02-07 12:39:02.207140542 +0000 UTC m=+5.547749645 container died aa986beb783de69f1a73e2979e5a7c1173c2deb5834e4933d86e0e3b958c1991 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-02-07T12:39:02.796 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:02 smithi174 podman[112244]: 2024-02-07 12:39:02.59323704 +0000 UTC m=+5.933846152 container cleanup aa986beb783de69f1a73e2979e5a7c1173c2deb5834e4933d86e0e3b958c1991 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux ) 2024-02-07T12:39:02.796 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:02 smithi174 bash[112244]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0 2024-02-07T12:39:03.141 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:02 smithi174 podman[112260]: 2024-02-07 12:39:02.823176128 +0000 UTC m=+0.616788811 container remove aa986beb783de69f1a73e2979e5a7c1173c2deb5834e4933d86e0e3b958c1991 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:39:03.457 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:03 smithi174 podman[112355]: 2024-02-07 12:39:03.042942897 +0000 UTC m=+0.018803482 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:39:03.457 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:03 smithi174 podman[112355]: 2024-02-07 12:39:03.170263745 +0000 UTC m=+0.146124317 container create 7b92b40dc08a7f532d4dd20461ddb23ae8d5c0122dc7bc7ddd585bb0d8819be8 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:39:03.458 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:03 smithi174 podman[112355]: 2024-02-07 12:39:03.381360538 +0000 UTC m=+0.357221113 container init 7b92b40dc08a7f532d4dd20461ddb23ae8d5c0122dc7bc7ddd585bb0d8819be8 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-02-07T12:39:03.458 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:03 smithi174 podman[112355]: 2024-02-07 12:39:03.384780599 +0000 UTC m=+0.360641175 container start 7b92b40dc08a7f532d4dd20461ddb23ae8d5c0122dc7bc7ddd585bb0d8819be8 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:39:03.458 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:03 smithi174 podman[112355]: 2024-02-07 12:39:03.4560259 +0000 UTC m=+0.431886475 container attach 7b92b40dc08a7f532d4dd20461ddb23ae8d5c0122dc7bc7ddd585bb0d8819be8 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, GIT_CLEAN=True, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:39:03.796 INFO:journalctl@ceph.osd.0.smithi174.stdout:Feb 07 12:39:03 smithi174 podman[112355]: 2024-02-07 12:39:03.687291873 +0000 UTC m=+0.663152444 container died 7b92b40dc08a7f532d4dd20461ddb23ae8d5c0122dc7bc7ddd585bb0d8819be8 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:39:04.267 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.0.service' 2024-02-07T12:39:04.307 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:39:04.308 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-02-07T12:39:04.308 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-02-07T12:39:04.308 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.1 2024-02-07T12:39:04.546 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:04 smithi174 systemd[1]: Stopping Ceph osd.1 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:39:05.046 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:04 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:39:04.636+0000 7f5ac1ec9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:39:05.046 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:04 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:39:04.636+0000 7f5ac1ec9700 -1 osd.1 158 *** Got signal Terminated *** 2024-02-07T12:39:05.046 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:04 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1[74982]: 2024-02-07T12:39:04.636+0000 7f5ac1ec9700 -1 osd.1 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:39:10.296 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:10 smithi174 podman[112460]: 2024-02-07 12:39:10.034764835 +0000 UTC m=+5.508614154 container died 6e9164f04adbbe31c2b5311c2ed35ae52263cd63f87e68d4e4680a8d913a24bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-02-07T12:39:10.631 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:10 smithi174 podman[112460]: 2024-02-07 12:39:10.387439764 +0000 UTC m=+5.861289083 container cleanup 6e9164f04adbbe31c2b5311c2ed35ae52263cd63f87e68d4e4680a8d913a24bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.2) 2024-02-07T12:39:10.632 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:10 smithi174 bash[112460]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1 2024-02-07T12:39:10.955 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:10 smithi174 podman[112479]: 2024-02-07 12:39:10.630038155 +0000 UTC m=+0.596525025 container remove 6e9164f04adbbe31c2b5311c2ed35ae52263cd63f87e68d4e4680a8d913a24bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.vendor=CentOS) 2024-02-07T12:39:11.296 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:10 smithi174 podman[112575]: 2024-02-07 12:39:10.856855673 +0000 UTC m=+0.017935904 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:39:11.296 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:10 smithi174 podman[112575]: 2024-02-07 12:39:10.964187492 +0000 UTC m=+0.125267758 container create c5160700c8741f05ae7aa645fd252236a2af5251da7099ce7d697fe1442e05ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:39:11.639 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:11 smithi174 podman[112575]: 2024-02-07 12:39:11.336893741 +0000 UTC m=+0.497973964 container init c5160700c8741f05ae7aa645fd252236a2af5251da7099ce7d697fe1442e05ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:39:11.639 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:11 smithi174 podman[112575]: 2024-02-07 12:39:11.34105989 +0000 UTC m=+0.502140117 container start c5160700c8741f05ae7aa645fd252236a2af5251da7099ce7d697fe1442e05ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-02-07T12:39:11.639 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:11 smithi174 podman[112575]: 2024-02-07 12:39:11.475241492 +0000 UTC m=+0.636321727 container attach c5160700c8741f05ae7aa645fd252236a2af5251da7099ce7d697fe1442e05ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-02-07T12:39:11.930 INFO:journalctl@ceph.osd.1.smithi174.stdout:Feb 07 12:39:11 smithi174 podman[112575]: 2024-02-07 12:39:11.637893773 +0000 UTC m=+0.798974006 container died c5160700c8741f05ae7aa645fd252236a2af5251da7099ce7d697fe1442e05ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:39:12.223 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.1.service' 2024-02-07T12:39:12.264 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:39:12.264 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-02-07T12:39:12.264 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-02-07T12:39:12.265 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.2 2024-02-07T12:39:12.546 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:12 smithi174 systemd[1]: Stopping Ceph osd.2 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:39:13.047 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[77701]: 2024-02-07T12:39:12.598+0000 7fa894ebc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:39:13.048 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[77701]: 2024-02-07T12:39:12.598+0000 7fa894ebc700 -1 osd.2 158 *** Got signal Terminated *** 2024-02-07T12:39:13.048 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:12 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2[77701]: 2024-02-07T12:39:12.598+0000 7fa894ebc700 -1 osd.2 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:39:18.277 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:17 smithi174 podman[112681]: 2024-02-07 12:39:17.98337624 +0000 UTC m=+5.496376423 container died 5dcb68e59527b2940793285ccbdd41be528954610a303ff6e45453bf731f568f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131) 2024-02-07T12:39:18.546 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:18 smithi174 podman[112681]: 2024-02-07 12:39:18.348136089 +0000 UTC m=+5.861136272 container cleanup 5dcb68e59527b2940793285ccbdd41be528954610a303ff6e45453bf731f568f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.2) 2024-02-07T12:39:18.546 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:18 smithi174 bash[112681]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2 2024-02-07T12:39:19.017 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:18 smithi174 podman[112730]: 2024-02-07 12:39:18.657496744 +0000 UTC m=+0.664635965 container remove 5dcb68e59527b2940793285ccbdd41be528954610a303ff6e45453bf731f568f (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-02-07T12:39:19.297 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:19 smithi174 podman[112823]: 2024-02-07 12:39:18.918522348 +0000 UTC m=+0.018288310 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:39:19.297 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:19 smithi174 podman[112823]: 2024-02-07 12:39:19.024835168 +0000 UTC m=+0.124601122 container create e8f522270e41aec76fd7c313a893f84a074cc4a43bd9fc43792bbaef24bb7111 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:39:19.297 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:19 smithi174 podman[112823]: 2024-02-07 12:39:19.227533073 +0000 UTC m=+0.327299036 container init e8f522270e41aec76fd7c313a893f84a074cc4a43bd9fc43792bbaef24bb7111 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-02-07T12:39:19.297 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:19 smithi174 podman[112823]: 2024-02-07 12:39:19.230944374 +0000 UTC m=+0.330710329 container start e8f522270e41aec76fd7c313a893f84a074cc4a43bd9fc43792bbaef24bb7111 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:39:19.797 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:19 smithi174 podman[112823]: 2024-02-07 12:39:19.321149698 +0000 UTC m=+0.420915674 container attach e8f522270e41aec76fd7c313a893f84a074cc4a43bd9fc43792bbaef24bb7111 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-02-07T12:39:19.797 INFO:journalctl@ceph.osd.2.smithi174.stdout:Feb 07 12:39:19 smithi174 podman[112823]: 2024-02-07 12:39:19.520347616 +0000 UTC m=+0.620113582 container died e8f522270e41aec76fd7c313a893f84a074cc4a43bd9fc43792bbaef24bb7111 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-02-07T12:39:20.116 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.2.service' 2024-02-07T12:39:20.156 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:39:20.157 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-02-07T12:39:20.157 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-02-07T12:39:20.157 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.3 2024-02-07T12:39:20.511 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:20 smithi174 systemd[1]: Stopping Ceph osd.3 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:39:20.796 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:20 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[80311]: 2024-02-07T12:39:20.508+0000 7f551a03c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:39:20.797 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:20 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[80311]: 2024-02-07T12:39:20.508+0000 7f551a03c700 -1 osd.3 158 *** Got signal Terminated *** 2024-02-07T12:39:20.797 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:20 smithi174 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3[80311]: 2024-02-07T12:39:20.508+0000 7f551a03c700 -1 osd.3 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:39:23.948 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:23 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:23.838+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:23.949 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:23 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:23.658+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:23.949 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:23 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:23.756+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:24.948 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:24 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:24.814+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:24.948 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:24 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:24.621+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:24.949 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:24 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:24.789+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:25.585 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:25 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:25.270+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:25.948 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:25 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:25.794+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:25.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:25 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:25.762+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:25.949 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:25 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:25.583+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:26.194 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:25 smithi174 podman[112928]: 2024-02-07 12:39:25.918367238 +0000 UTC m=+5.540022642 container died ea1ed26d44767b80a123e22530a2108d537fa9740027bd00ad5dcd4073bad54d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-02-07T12:39:26.544 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:26 smithi174 podman[112928]: 2024-02-07 12:39:26.274849913 +0000 UTC m=+5.896505315 container cleanup ea1ed26d44767b80a123e22530a2108d537fa9740027bd00ad5dcd4073bad54d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:39:26.544 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:26 smithi174 bash[112928]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3 2024-02-07T12:39:26.632 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:26 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:26.224+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:26.796 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:26 smithi174 podman[112945]: 2024-02-07 12:39:26.542622866 +0000 UTC m=+0.619739065 container remove ea1ed26d44767b80a123e22530a2108d537fa9740027bd00ad5dcd4073bad54d (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-02-07T12:39:26.948 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:26 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:26.793+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:26.948 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:26 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:26.631+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:26.949 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:26 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:26.811+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:27.122 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:26 smithi174 podman[113042]: 2024-02-07 12:39:26.851689478 +0000 UTC m=+0.106631767 container create ae7e27bfa6ae7acc9b5bca5d26bb75d847cfd8dadce2cc12a02e28027fe8a1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240131) 2024-02-07T12:39:27.122 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:26 smithi174 podman[113042]: 2024-02-07 12:39:26.763759266 +0000 UTC m=+0.018701524 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:39:27.122 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:27 smithi174 podman[113042]: 2024-02-07 12:39:27.028881483 +0000 UTC m=+0.283823727 container init ae7e27bfa6ae7acc9b5bca5d26bb75d847cfd8dadce2cc12a02e28027fe8a1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:39:27.122 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:27 smithi174 podman[113042]: 2024-02-07 12:39:27.03197918 +0000 UTC m=+0.286921425 container start ae7e27bfa6ae7acc9b5bca5d26bb75d847cfd8dadce2cc12a02e28027fe8a1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True) 2024-02-07T12:39:27.122 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:27 smithi174 podman[113042]: 2024-02-07 12:39:27.120586519 +0000 UTC m=+0.375528793 container attach ae7e27bfa6ae7acc9b5bca5d26bb75d847cfd8dadce2cc12a02e28027fe8a1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:39:27.546 INFO:journalctl@ceph.osd.3.smithi174.stdout:Feb 07 12:39:27 smithi174 podman[113042]: 2024-02-07 12:39:27.319179805 +0000 UTC m=+0.574122044 container died ae7e27bfa6ae7acc9b5bca5d26bb75d847cfd8dadce2cc12a02e28027fe8a1d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-3-deactivate, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131) 2024-02-07T12:39:27.621 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:27 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:27.208+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:27.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:27 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:27.819+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:27.948 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:27 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:27.821+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:27.949 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:27 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:27.619+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:27.960 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.3.service' 2024-02-07T12:39:28.003 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:39:28.003 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-02-07T12:39:28.003 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-02-07T12:39:28.003 DEBUG:teuthology.orchestra.run.smithi184:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.4 2024-02-07T12:39:28.319 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:28 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:28.220+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:28.319 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:28 smithi184 systemd[1]: Stopping Ceph osd.4 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:39:28.319 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:28 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:28.316+0000 7fafea806700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:39:28.578 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:28 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:28.316+0000 7fafea806700 -1 osd.4 158 *** Got signal Terminated *** 2024-02-07T12:39:28.578 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:28 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:28.316+0000 7fafea806700 -1 osd.4 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:39:28.865 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:28 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:28.576+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:28.866 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:28 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:28.825+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:29.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:28 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:28.863+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:29.569 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:29 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:29.200+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:29.882 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:29 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:29.567+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:29.882 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:29 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:29.806+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:30.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:29 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:29.880+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:30.519 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:30 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:30.236+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:30.854 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:30 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:30.518+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:31.198 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:30 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:30.852+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:31.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:30 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:30.880+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:31.563 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:31 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:31.245+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:31.860 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:31 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:31.561+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:32.198 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:31 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:31.874+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:32.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:31 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:31.859+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:32.520 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:32.255+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:32.860 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:32.519+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:33.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:32.882+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:33.198 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:32.858+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.571760+0000 front 2024-02-07T12:38:57.571742+0000 (oldest deadline 2024-02-07T12:39:23.471674+0000) 2024-02-07T12:39:33.198 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:32 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4[57999]: 2024-02-07T12:39:32.858+0000 7fafe2df9700 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.273188+0000 front 2024-02-07T12:39:09.273277+0000 (oldest deadline 2024-02-07T12:39:32.773101+0000) 2024-02-07T12:39:33.520 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:33 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:33.262+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:33.521 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:33 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:33.262+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:33.871 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:33 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:33.519+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:34.153 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:33 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:33.869+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:34.154 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:33 smithi184 podman[78496]: 2024-02-07 12:39:33.874433554 +0000 UTC m=+5.680893322 container died 2c6c37d4be2a983b588e0555bcfbbfe252292f0fd408d81141d6fd8549aeec20 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131) 2024-02-07T12:39:34.448 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:34 smithi184 podman[78496]: 2024-02-07 12:39:34.334525571 +0000 UTC m=+6.140985338 container cleanup 2c6c37d4be2a983b588e0555bcfbbfe252292f0fd408d81141d6fd8549aeec20 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131) 2024-02-07T12:39:34.448 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:34 smithi184 bash[78496]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4 2024-02-07T12:39:34.448 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:34 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:34.308+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:34.448 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:34 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:34.308+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:34.840 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:34 smithi184 podman[78513]: 2024-02-07 12:39:34.594278194 +0000 UTC m=+0.714084189 container remove 2c6c37d4be2a983b588e0555bcfbbfe252292f0fd408d81141d6fd8549aeec20 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-02-07T12:39:34.841 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:34 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:34.499+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:34.841 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:34 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:34.499+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.236381+0000 front 2024-02-07T12:39:09.236440+0000 (oldest deadline 2024-02-07T12:39:33.936484+0000) 2024-02-07T12:39:35.101 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:34 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:34.838+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:35.102 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:34 smithi184 podman[78607]: 2024-02-07 12:39:34.857799894 +0000 UTC m=+0.018027599 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:39:35.102 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:34 smithi184 podman[78607]: 2024-02-07 12:39:34.961843524 +0000 UTC m=+0.122071216 container create df50a67712b15de743b7c46ac1fb568fbee34c1d5a107432cf899ad096245d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_CLEAN=True, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:39:35.448 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:35 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:35.329+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:35.448 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:35 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:35.329+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:35.448 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:35 smithi184 podman[78607]: 2024-02-07 12:39:35.214730644 +0000 UTC m=+0.374958334 container init df50a67712b15de743b7c46ac1fb568fbee34c1d5a107432cf899ad096245d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, CEPH_POINT_RELEASE=) 2024-02-07T12:39:35.448 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:35 smithi184 podman[78607]: 2024-02-07 12:39:35.217829703 +0000 UTC m=+0.378057403 container start df50a67712b15de743b7c46ac1fb568fbee34c1d5a107432cf899ad096245d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_BRANCH=HEAD) 2024-02-07T12:39:35.449 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:35 smithi184 podman[78607]: 2024-02-07 12:39:35.29787155 +0000 UTC m=+0.458099249 container attach df50a67712b15de743b7c46ac1fb568fbee34c1d5a107432cf899ad096245d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-02-07T12:39:35.811 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:35 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:35.490+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:35.811 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:35 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:35.490+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.236381+0000 front 2024-02-07T12:39:09.236440+0000 (oldest deadline 2024-02-07T12:39:33.936484+0000) 2024-02-07T12:39:35.811 INFO:journalctl@ceph.osd.4.smithi184.stdout:Feb 07 12:39:35 smithi184 podman[78607]: 2024-02-07 12:39:35.514538078 +0000 UTC m=+0.674765769 container died df50a67712b15de743b7c46ac1fb568fbee34c1d5a107432cf899ad096245d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-4-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-02-07T12:39:36.087 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:35 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:35.869+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:36.087 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:35 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:35.869+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:36.115 DEBUG:teuthology.orchestra.run.smithi184:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.4.service' 2024-02-07T12:39:36.156 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:39:36.156 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-02-07T12:39:36.156 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-02-07T12:39:36.157 DEBUG:teuthology.orchestra.run.smithi184:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.5 2024-02-07T12:39:36.360 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:36 smithi184 systemd[1]: Stopping Ceph osd.5 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:39:36.361 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:36.317+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:36.361 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:36.317+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:36.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:36.468+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:36.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:36.468+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.236381+0000 front 2024-02-07T12:39:09.236440+0000 (oldest deadline 2024-02-07T12:39:33.936484+0000) 2024-02-07T12:39:36.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:36.521+0000 7f9da6198700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:39:36.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:36.521+0000 7f9da6198700 -1 osd.5 158 *** Got signal Terminated *** 2024-02-07T12:39:36.699 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:36.521+0000 7f9da6198700 -1 osd.5 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:39:37.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:36.857+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:37.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:36 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:36.857+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:37.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:37.319+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:37.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:37.319+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:37.699 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:37.492+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:37.699 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:37.492+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.236381+0000 front 2024-02-07T12:39:09.236440+0000 (oldest deadline 2024-02-07T12:39:33.936484+0000) 2024-02-07T12:39:38.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:37.812+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:38.200 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:37 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:37.812+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:38.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:38 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:38.298+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:38.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:38 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:38.298+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:38.699 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:38 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:38.513+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:38.699 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:38 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:38.513+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.236381+0000 front 2024-02-07T12:39:09.236440+0000 (oldest deadline 2024-02-07T12:39:33.936484+0000) 2024-02-07T12:39:39.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:38 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:38.851+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:39.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:38 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:38.851+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:39.563 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:39 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:39.266+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:39.563 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:39 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:39.266+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:39.854 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:39 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:39.560+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:39.854 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:39 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:39.560+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.236381+0000 front 2024-02-07T12:39:09.236440+0000 (oldest deadline 2024-02-07T12:39:33.936484+0000) 2024-02-07T12:39:39.854 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:39 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:39.560+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.637565+0000 front 2024-02-07T12:39:15.637703+0000 (oldest deadline 2024-02-07T12:39:38.537554+0000) 2024-02-07T12:39:40.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:39 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:39.851+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:40.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:39 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:39.851+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:40.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:39 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:39.851+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:40.525 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:40.251+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:40.525 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:40.251+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:40.525 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:40.251+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:40.888 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:40.522+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:40.888 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:40.522+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.236381+0000 front 2024-02-07T12:39:09.236440+0000 (oldest deadline 2024-02-07T12:39:33.936484+0000) 2024-02-07T12:39:40.888 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:40.522+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.637565+0000 front 2024-02-07T12:39:15.637703+0000 (oldest deadline 2024-02-07T12:39:38.537554+0000) 2024-02-07T12:39:41.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:40.886+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:41.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:40.886+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:41.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:40 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:40.886+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:41.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:41.282+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:41.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:41.282+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:41.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:41.282+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:41.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:41.475+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.634349+0000 front 2024-02-07T12:38:57.634410+0000 (oldest deadline 2024-02-07T12:39:22.934286+0000) 2024-02-07T12:39:41.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:41.475+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.236381+0000 front 2024-02-07T12:39:09.236440+0000 (oldest deadline 2024-02-07T12:39:33.936484+0000) 2024-02-07T12:39:41.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5[60255]: 2024-02-07T12:39:41.475+0000 7f9da2793700 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.637565+0000 front 2024-02-07T12:39:15.637703+0000 (oldest deadline 2024-02-07T12:39:38.537554+0000) 2024-02-07T12:39:42.198 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:42 smithi184 podman[78745]: 2024-02-07 12:39:42.039873084 +0000 UTC m=+5.641247004 container died 8b5693e2bfd0d830a564626ec9dc97e6f69c37de05acd323a7cc0336353eb4eb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, ceph=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-02-07T12:39:42.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:41.846+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:42.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:41.846+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:42.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:41 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:41.846+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:42.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:42 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:42.301+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:42.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:42 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:42.302+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:42.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:42 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:42.302+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:42.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:42 smithi184 podman[78745]: 2024-02-07 12:39:42.388298729 +0000 UTC m=+5.989672648 container cleanup 8b5693e2bfd0d830a564626ec9dc97e6f69c37de05acd323a7cc0336353eb4eb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:39:42.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:42 smithi184 bash[78745]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5 2024-02-07T12:39:43.056 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:42 smithi184 podman[78773]: 2024-02-07 12:39:42.714774849 +0000 UTC m=+0.670870326 container remove 8b5693e2bfd0d830a564626ec9dc97e6f69c37de05acd323a7cc0336353eb4eb (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240131, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-02-07T12:39:43.056 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:42 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:42.812+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:43.056 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:42 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:42.812+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:43.056 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:42 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:42.812+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:43.336 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:43 smithi184 podman[78864]: 2024-02-07 12:39:42.95689861 +0000 UTC m=+0.018640436 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:39:43.336 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:43 smithi184 podman[78864]: 2024-02-07 12:39:43.065668145 +0000 UTC m=+0.127410011 container create ec1de7585114bd736f47fd295b55cf1c1887106ff80d39ae324974a2d4bcb4ff (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240131) 2024-02-07T12:39:43.336 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:43 smithi184 podman[78864]: 2024-02-07 12:39:43.277297014 +0000 UTC m=+0.339038839 container init ec1de7585114bd736f47fd295b55cf1c1887106ff80d39ae324974a2d4bcb4ff (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0) 2024-02-07T12:39:43.336 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:43 smithi184 podman[78864]: 2024-02-07 12:39:43.28108351 +0000 UTC m=+0.342825339 container start ec1de7585114bd736f47fd295b55cf1c1887106ff80d39ae324974a2d4bcb4ff (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, ceph=True, org.label-schema.build-date=20240131, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, org.label-schema.schema-version=1.0) 2024-02-07T12:39:43.337 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:43 smithi184 podman[78864]: 2024-02-07 12:39:43.334822647 +0000 UTC m=+0.396564486 container attach ec1de7585114bd736f47fd295b55cf1c1887106ff80d39ae324974a2d4bcb4ff (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, io.buildah.version=1.33.2, GIT_CLEAN=True, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS) 2024-02-07T12:39:43.698 INFO:journalctl@ceph.osd.5.smithi184.stdout:Feb 07 12:39:43 smithi184 podman[78864]: 2024-02-07 12:39:43.571932916 +0000 UTC m=+0.633674743 container died ec1de7585114bd736f47fd295b55cf1c1887106ff80d39ae324974a2d4bcb4ff (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2) 2024-02-07T12:39:43.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:43 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:43.349+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:43.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:43 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:43.349+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:43.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:43 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:43.349+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:44.181 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:43 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:43.851+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:44.181 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:43 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:43.851+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:44.182 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:43 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:43.851+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:44.196 DEBUG:teuthology.orchestra.run.smithi184:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.5.service' 2024-02-07T12:39:44.236 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:39:44.236 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-02-07T12:39:44.236 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-02-07T12:39:44.237 DEBUG:teuthology.orchestra.run.smithi184:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.6 2024-02-07T12:39:44.448 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:44 smithi184 systemd[1]: Stopping Ceph osd.6 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:39:44.448 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:44.328+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:44.449 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:44.328+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:44.449 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:44.328+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:44.830 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:44.558+0000 7fb3cc402700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:39:44.830 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:44.558+0000 7fb3cc402700 -1 osd.6 158 *** Got signal Terminated *** 2024-02-07T12:39:44.830 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:44.558+0000 7fb3cc402700 -1 osd.6 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:39:44.831 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:44.828+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:44.831 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:44.828+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:44.831 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:44 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:44.828+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:45.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:45 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:45.333+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:45.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:45 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:45.333+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:45.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:45 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:45.333+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:46.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:45 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:45.838+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:46.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:45 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:45.838+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:46.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:45 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:45.838+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:46.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:46 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:46.346+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:46.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:46 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:46.346+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:46.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:46 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:46.346+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:47.199 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:46 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:46.849+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:47.200 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:46 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:46.849+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:47.200 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:46 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:46.849+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:47.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:47 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:47.318+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:47.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:47 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:47.319+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:47.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:47 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:47.319+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:48.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:47 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:47.806+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:48.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:47 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:47.806+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:48.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:47 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:47.806+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:48.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:48 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:48.292+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:48.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:48 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:48.292+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:48.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:48 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:48.292+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:49.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:48 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:48.821+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:57.721554+0000 front 2024-02-07T12:38:57.721507+0000 (oldest deadline 2024-02-07T12:39:23.521725+0000) 2024-02-07T12:39:49.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:48 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:48.821+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.422806+0000 front 2024-02-07T12:39:09.422951+0000 (oldest deadline 2024-02-07T12:39:35.223025+0000) 2024-02-07T12:39:49.198 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:48 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6[62511]: 2024-02-07T12:39:48.821+0000 7fb3c49f5700 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:16.824255+0000 front 2024-02-07T12:39:16.824269+0000 (oldest deadline 2024-02-07T12:39:39.724133+0000) 2024-02-07T12:39:49.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:49 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:49.339+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:49.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:49 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:49.339+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:49.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:49 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:49.339+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:49.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:49 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:49.339+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6830 osd.3 since back 2024-02-07T12:39:23.476459+0000 front 2024-02-07T12:39:23.476440+0000 (oldest deadline 2024-02-07T12:39:48.676741+0000) 2024-02-07T12:39:50.326 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:50 smithi184 podman[78967]: 2024-02-07 12:39:50.037514122 +0000 UTC m=+5.583974753 container died 653dd44c30002917ef2b46e193268d3844ae204c79e4eb610f5bbdfab4182fce (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-02-07T12:39:50.660 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:50 smithi184 podman[78967]: 2024-02-07 12:39:50.42447067 +0000 UTC m=+5.970931302 container cleanup 653dd44c30002917ef2b46e193268d3844ae204c79e4eb610f5bbdfab4182fce (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:39:50.661 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:50 smithi184 bash[78967]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6 2024-02-07T12:39:50.661 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:50 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:50.353+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:50.661 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:50 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:50.353+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:50.661 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:50 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:50.353+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:50.661 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:50 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:50.353+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6830 osd.3 since back 2024-02-07T12:39:23.476459+0000 front 2024-02-07T12:39:23.476440+0000 (oldest deadline 2024-02-07T12:39:48.676741+0000) 2024-02-07T12:39:50.948 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:50 smithi184 podman[78984]: 2024-02-07 12:39:50.659028387 +0000 UTC m=+0.618000081 container remove 653dd44c30002917ef2b46e193268d3844ae204c79e4eb610f5bbdfab4182fce (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.build-date=20240131) 2024-02-07T12:39:51.238 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:50 smithi184 podman[79073]: 2024-02-07 12:39:50.968299757 +0000 UTC m=+0.101533689 container create bac3d0e7b6ff963e8ff41ff0f40377c4496670caa1786018caa0c4989c5fc68b (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-02-07T12:39:51.239 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:50 smithi184 podman[79073]: 2024-02-07 12:39:50.885101521 +0000 UTC m=+0.018335410 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:39:51.239 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:51 smithi184 podman[79073]: 2024-02-07 12:39:51.163067226 +0000 UTC m=+0.296301103 container init bac3d0e7b6ff963e8ff41ff0f40377c4496670caa1786018caa0c4989c5fc68b (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux ) 2024-02-07T12:39:51.239 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:51 smithi184 podman[79073]: 2024-02-07 12:39:51.166404963 +0000 UTC m=+0.299638847 container start bac3d0e7b6ff963e8ff41ff0f40377c4496670caa1786018caa0c4989c5fc68b (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:39:51.239 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:51 smithi184 podman[79073]: 2024-02-07 12:39:51.237155835 +0000 UTC m=+0.370389730 container attach bac3d0e7b6ff963e8ff41ff0f40377c4496670caa1786018caa0c4989c5fc68b (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-02-07T12:39:51.698 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:51 smithi184 podman[79073]: 2024-02-07 12:39:51.460029275 +0000 UTC m=+0.593263154 container died bac3d0e7b6ff963e8ff41ff0f40377c4496670caa1786018caa0c4989c5fc68b (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, ceph=True, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.vendor=CentOS, io.buildah.version=1.33.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-02-07T12:39:51.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:51 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:51.387+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:51.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:51 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:51.387+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:51.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:51 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:51.387+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:51.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:51 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:51.387+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6830 osd.3 since back 2024-02-07T12:39:23.476459+0000 front 2024-02-07T12:39:23.476440+0000 (oldest deadline 2024-02-07T12:39:48.676741+0000) 2024-02-07T12:39:52.009 INFO:journalctl@ceph.osd.6.smithi184.stdout:Feb 07 12:39:51 smithi184 podman[79099]: 2024-02-07 12:39:51.990801997 +0000 UTC m=+0.524806493 container remove bac3d0e7b6ff963e8ff41ff0f40377c4496670caa1786018caa0c4989c5fc68b (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240131, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:39:52.015 DEBUG:teuthology.orchestra.run.smithi184:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.6.service' 2024-02-07T12:39:52.057 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:39:52.057 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-02-07T12:39:52.057 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-02-07T12:39:52.057 DEBUG:teuthology.orchestra.run.smithi184:> sudo systemctl stop ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.7 2024-02-07T12:39:52.371 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:52 smithi184 systemd[1]: Stopping Ceph osd.7 for 53d92d24-c5b2-11ee-95b6-87774f69a715... 2024-02-07T12:39:52.371 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:52 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:52.368+0000 7f78b6363700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-02-07T12:39:52.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:52 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:52.368+0000 7f78b6363700 -1 osd.7 158 *** Got signal Terminated *** 2024-02-07T12:39:52.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:52 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:52.368+0000 7f78b6363700 -1 osd.7 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-02-07T12:39:52.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:52 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:52.381+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:52.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:52 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:52.381+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:52.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:52 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:52.381+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:52.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:52 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:52.381+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6830 osd.3 since back 2024-02-07T12:39:23.476459+0000 front 2024-02-07T12:39:23.476440+0000 (oldest deadline 2024-02-07T12:39:48.676741+0000) 2024-02-07T12:39:53.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:53 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:53.393+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:53.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:53 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:53.393+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:53.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:53 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:53.393+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:53.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:53 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:53.393+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6830 osd.3 since back 2024-02-07T12:39:23.476459+0000 front 2024-02-07T12:39:23.476440+0000 (oldest deadline 2024-02-07T12:39:48.676741+0000) 2024-02-07T12:39:54.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:54 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:54.423+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:54.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:54 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:54.423+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:54.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:54 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:54.423+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:54.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:54 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:54.423+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6830 osd.3 since back 2024-02-07T12:39:23.476459+0000 front 2024-02-07T12:39:23.476440+0000 (oldest deadline 2024-02-07T12:39:48.676741+0000) 2024-02-07T12:39:55.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:55 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:55.424+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:55.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:55 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:55.424+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:55.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:55 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:55.424+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:55.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:55 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:55.424+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6830 osd.3 since back 2024-02-07T12:39:23.476459+0000 front 2024-02-07T12:39:23.476440+0000 (oldest deadline 2024-02-07T12:39:48.676741+0000) 2024-02-07T12:39:55.699 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:55 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:55.424+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.184:6804 osd.4 since back 2024-02-07T12:39:32.177685+0000 front 2024-02-07T12:39:32.177789+0000 (oldest deadline 2024-02-07T12:39:54.477731+0000) 2024-02-07T12:39:56.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:56 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:56.445+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6806 osd.0 since back 2024-02-07T12:38:56.072828+0000 front 2024-02-07T12:38:56.072715+0000 (oldest deadline 2024-02-07T12:39:24.273338+0000) 2024-02-07T12:39:56.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:56 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:56.445+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6814 osd.1 since back 2024-02-07T12:39:09.474859+0000 front 2024-02-07T12:39:09.474886+0000 (oldest deadline 2024-02-07T12:39:32.374748+0000) 2024-02-07T12:39:56.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:56 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:56.445+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6822 osd.2 since back 2024-02-07T12:39:15.275652+0000 front 2024-02-07T12:39:15.275563+0000 (oldest deadline 2024-02-07T12:39:39.375455+0000) 2024-02-07T12:39:56.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:56 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:56.445+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.174:6830 osd.3 since back 2024-02-07T12:39:23.476459+0000 front 2024-02-07T12:39:23.476440+0000 (oldest deadline 2024-02-07T12:39:48.676741+0000) 2024-02-07T12:39:56.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:56 smithi184 ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7[64762]: 2024-02-07T12:39:56.445+0000 7f78b295e700 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.184:6804 osd.4 since back 2024-02-07T12:39:32.177685+0000 front 2024-02-07T12:39:32.177789+0000 (oldest deadline 2024-02-07T12:39:54.477731+0000) 2024-02-07T12:39:58.212 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:57 smithi184 podman[79179]: 2024-02-07 12:39:57.954520793 +0000 UTC m=+5.675322055 container died 0445d4324ff0ca8bb5d28e642a6bf5bbd5b133f33c733fec2c6bb2ffb2899801 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240131) 2024-02-07T12:39:58.613 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:58 smithi184 podman[79179]: 2024-02-07 12:39:58.385207173 +0000 UTC m=+6.106008498 container cleanup 0445d4324ff0ca8bb5d28e642a6bf5bbd5b133f33c733fec2c6bb2ffb2899801 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240131, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275) 2024-02-07T12:39:58.613 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:58 smithi184 bash[79179]: ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7 2024-02-07T12:39:58.948 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:58 smithi184 podman[79241]: 2024-02-07 12:39:58.611507548 +0000 UTC m=+0.652802018 container remove 0445d4324ff0ca8bb5d28e642a6bf5bbd5b133f33c733fec2c6bb2ffb2899801 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-02-07T12:39:59.349 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:59 smithi184 podman[79332]: 2024-02-07 12:39:59.045613902 +0000 UTC m=+0.115295502 container create 8949c4edc627eaba4bdf6cb05393d90dc41eb3ae29385055ca5803a8f888fc82 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, org.label-schema.build-date=20240131, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.2, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-02-07T12:39:59.350 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:59 smithi184 podman[79332]: 2024-02-07 12:39:58.949258479 +0000 UTC m=+0.018940080 image pull 79957b3723074dc4d7b43e7345862a464daec6a6b69febcfe25c399b70e0301f quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe 2024-02-07T12:39:59.350 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:59 smithi184 podman[79332]: 2024-02-07 12:39:59.248209063 +0000 UTC m=+0.317890677 container init 8949c4edc627eaba4bdf6cb05393d90dc41eb3ae29385055ca5803a8f888fc82 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, org.label-schema.build-date=20240131, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.2, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-02-07T12:39:59.350 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:59 smithi184 podman[79332]: 2024-02-07 12:39:59.251436732 +0000 UTC m=+0.321118338 container start 8949c4edc627eaba4bdf6cb05393d90dc41eb3ae29385055ca5803a8f888fc82 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240131, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc) 2024-02-07T12:39:59.350 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:59 smithi184 podman[79332]: 2024-02-07 12:39:59.348071434 +0000 UTC m=+0.417753039 container attach 8949c4edc627eaba4bdf6cb05393d90dc41eb3ae29385055ca5803a8f888fc82 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.2, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-02-07T12:39:59.698 INFO:journalctl@ceph.osd.7.smithi184.stdout:Feb 07 12:39:59 smithi184 podman[79332]: 2024-02-07 12:39:59.54237329 +0000 UTC m=+0.612054894 container died 8949c4edc627eaba4bdf6cb05393d90dc41eb3ae29385055ca5803a8f888fc82 (image=quay.ceph.io/ceph-ci/ceph@sha256:c53d1d048facb107ddbb5178fea4d5ae496b0f9f9f34c5f63d4bece0be7a8bbe, name=ceph-53d92d24-c5b2-11ee-95b6-87774f69a715-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=14dafef590928bb89728de91784b87b8f9bbf275, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-4-2024-02-05-1854-44458cc, io.buildah.version=1.33.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240131, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-02-07T12:40:00.108 DEBUG:teuthology.orchestra.run.smithi184:> sudo pkill -f 'journalctl -f -n 0 -u ceph-53d92d24-c5b2-11ee-95b6-87774f69a715@osd.7.service' 2024-02-07T12:40:00.149 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-02-07T12:40:00.149 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-02-07T12:40:00.149 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 --force --keep-logs 2024-02-07T12:42:58.215 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 --force --keep-logs 2024-02-07T12:43:17.005 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:43:17.041 DEBUG:teuthology.orchestra.run.smithi184:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-02-07T12:43:17.070 INFO:tasks.cephadm:Archiving crash dumps... 2024-02-07T12:43:17.072 DEBUG:teuthology.misc:Transferring archived files from smithi174:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/crash to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725/remote/smithi174/crash 2024-02-07T12:43:17.073 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/crash -- . 2024-02-07T12:43:17.288 INFO:teuthology.orchestra.run.smithi174.stderr:tar: /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/crash: Cannot open: No such file or directory 2024-02-07T12:43:17.288 INFO:teuthology.orchestra.run.smithi174.stderr:tar: Error is not recoverable: exiting now 2024-02-07T12:43:17.290 DEBUG:teuthology.misc:Transferring archived files from smithi184:/var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/crash to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725/remote/smithi184/crash 2024-02-07T12:43:17.291 DEBUG:teuthology.orchestra.run.smithi184:> sudo tar c -f - -C /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/crash -- . 2024-02-07T12:43:17.336 INFO:teuthology.orchestra.run.smithi184.stderr:tar: /var/lib/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/crash: Cannot open: No such file or directory 2024-02-07T12:43:17.336 INFO:teuthology.orchestra.run.smithi184.stderr:tar: Error is not recoverable: exiting now 2024-02-07T12:43:17.337 INFO:tasks.cephadm:Checking cluster log for badness... 2024-02-07T12:43:17.337 DEBUG:teuthology.orchestra.run.smithi174:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-07T12:43:17.369 INFO:teuthology.orchestra.run.smithi174.stdout:2024-02-07T12:19:28.182128+0000 mon.smithi174 (mon.0) 491 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:43:17.370 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-02-07T12:43:17.370 DEBUG:teuthology.orchestra.run.smithi174:> sudo egrep '\[SEC\]' /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-07T12:43:17.442 DEBUG:teuthology.orchestra.run.smithi174:> sudo egrep '\[ERR\]' /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-07T12:43:17.513 DEBUG:teuthology.orchestra.run.smithi174:> sudo egrep '\[WRN\]' /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-02-07T12:43:17.583 INFO:teuthology.orchestra.run.smithi174.stdout:2024-02-07T12:19:28.182128+0000 mon.smithi174 (mon.0) 491 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-02-07T12:43:17.584 INFO:tasks.cephadm:Compressing logs... 2024-02-07T12:43:17.584 DEBUG:teuthology.orchestra.run.smithi174:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-02-07T12:43:17.627 DEBUG:teuthology.orchestra.run.smithi184:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-02-07T12:43:17.652 INFO:teuthology.orchestra.run.smithi174.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-02-07T12:43:17.652 INFO:teuthology.orchestra.run.smithi174.stderr:: No such file or directory 2024-02-07T12:43:17.653 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mgr.smithi174.whsozo.log 2024-02-07T12:43:17.653 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.audit.log 2024-02-07T12:43:17.653 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.1.log 2024-02-07T12:43:17.654 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mgr.smithi174.whsozo.log: /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mon.smithi174.log 2024-02-07T12:43:17.655 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.3.log 2024-02-07T12:43:17.655 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.1.log: /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mon.smithi174.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mds.foofs.smithi174.vkrzkp.log 2024-02-07T12:43:17.655 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.0.log 2024-02-07T12:43:17.655 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mds.foofs.smithi174.vkrzkp.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log 2024-02-07T12:43:17.655 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.0.log: 89.6% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mds.foofs.smithi174.vkrzkp.log.gz 2024-02-07T12:43:17.656 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log 2024-02-07T12:43:17.657 INFO:teuthology.orchestra.run.smithi184.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-02-07T12:43:17.657 INFO:teuthology.orchestra.run.smithi184.stderr:: No such file or directory 2024-02-07T12:43:17.657 INFO:teuthology.orchestra.run.smithi184.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.6.log 2024-02-07T12:43:17.657 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.audit.log 2024-02-07T12:43:17.657 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.4.log 2024-02-07T12:43:17.659 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log.gz 2024-02-07T12:43:17.659 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.2.log 2024-02-07T12:43:17.659 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log: 91.5% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.audit.log.gz 2024-02-07T12:43:17.663 INFO:teuthology.orchestra.run.smithi174.stderr: 87.6% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log.gz 2024-02-07T12:43:17.663 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-volume.log 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mds.foofs.smithi184.alpixp.log 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.7.log 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.4.log: /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mds.foofs.smithi184.alpixp.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.5.log 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mon.smithi184.log 87.8% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mds.foofs.smithi184.alpixp.log.gz 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr: 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mon.smithi184.log: gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mgr.smithi184.prgadw.log 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log: 85.6% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.cephadm.log.gz 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mgr.smithi184.prgadw.log: 91.6% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.audit.log.gz 2024-02-07T12:43:17.664 INFO:teuthology.orchestra.run.smithi184.stderr:gzip -5 --verbose -- /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-volume.log 2024-02-07T12:43:17.665 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log: 90.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-02-07T12:43:17.665 INFO:teuthology.orchestra.run.smithi184.stderr: 87.6% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph.log.gz 2024-02-07T12:43:17.672 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.2.log: /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-volume.log: 89.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-02-07T12:43:17.680 INFO:teuthology.orchestra.run.smithi174.stderr: 91.4% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-volume.log.gz 2024-02-07T12:43:17.681 INFO:teuthology.orchestra.run.smithi184.stderr:/var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-volume.log: 91.4% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-volume.log.gz 2024-02-07T12:43:17.708 INFO:teuthology.orchestra.run.smithi184.stderr: 89.1% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mgr.smithi184.prgadw.log.gz 2024-02-07T12:43:18.047 INFO:teuthology.orchestra.run.smithi174.stderr: 88.3% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mgr.smithi174.whsozo.log.gz 2024-02-07T12:43:18.164 INFO:teuthology.orchestra.run.smithi184.stderr: 92.6% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mon.smithi184.log.gz 2024-02-07T12:43:19.697 INFO:teuthology.orchestra.run.smithi174.stderr: 92.0% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-mon.smithi174.log.gz 2024-02-07T12:43:30.173 INFO:teuthology.orchestra.run.smithi174.stderr: 94.5% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.2.log.gz 2024-02-07T12:43:33.642 INFO:teuthology.orchestra.run.smithi184.stderr: 93.9% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.4.log.gz 2024-02-07T12:43:34.833 INFO:teuthology.orchestra.run.smithi184.stderr: 94.2% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.6.log.gz 2024-02-07T12:43:36.068 INFO:teuthology.orchestra.run.smithi174.stderr: 94.1% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.0.log.gz 2024-02-07T12:43:36.460 INFO:teuthology.orchestra.run.smithi174.stderr: 94.2% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.1.log.gz 2024-02-07T12:43:37.985 INFO:teuthology.orchestra.run.smithi174.stderr: 94.1% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.3.log.gz 2024-02-07T12:43:37.987 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-02-07T12:43:37.987 INFO:teuthology.orchestra.run.smithi174.stderr:real 0m20.348s 2024-02-07T12:43:37.987 INFO:teuthology.orchestra.run.smithi174.stderr:user 1m9.712s 2024-02-07T12:43:37.987 INFO:teuthology.orchestra.run.smithi174.stderr:sys 0m2.731s 2024-02-07T12:43:38.360 INFO:teuthology.orchestra.run.smithi184.stderr: 94.1% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.5.log.gz 2024-02-07T12:43:43.640 INFO:teuthology.orchestra.run.smithi184.stderr: 94.0% -- replaced with /var/log/ceph/53d92d24-c5b2-11ee-95b6-87774f69a715/ceph-osd.7.log.gz 2024-02-07T12:43:43.641 INFO:teuthology.orchestra.run.smithi184.stderr: 2024-02-07T12:43:43.641 INFO:teuthology.orchestra.run.smithi184.stderr:real 0m26.000s 2024-02-07T12:43:43.642 INFO:teuthology.orchestra.run.smithi184.stderr:user 1m17.387s 2024-02-07T12:43:43.642 INFO:teuthology.orchestra.run.smithi184.stderr:sys 0m2.923s 2024-02-07T12:43:43.642 INFO:tasks.cephadm:Archiving logs... 2024-02-07T12:43:43.642 DEBUG:teuthology.misc:Transferring archived files from smithi174:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725/remote/smithi174/log 2024-02-07T12:43:43.643 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /var/log/ceph -- . 2024-02-07T12:43:53.400 DEBUG:teuthology.misc:Transferring archived files from smithi184:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725/remote/smithi184/log 2024-02-07T12:43:53.401 DEBUG:teuthology.orchestra.run.smithi184:> sudo tar c -f - -C /var/log/ceph -- . 2024-02-07T12:44:04.864 INFO:tasks.cephadm:Removing cluster... 2024-02-07T12:44:05.036 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 --force 2024-02-07T12:44:05.411 DEBUG:teuthology.orchestra.run.smithi184:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 53d92d24-c5b2-11ee-95b6-87774f69a715 --force 2024-02-07T12:44:05.816 INFO:tasks.cephadm:Removing cephadm ... 2024-02-07T12:44:05.816 DEBUG:teuthology.orchestra.run.smithi174:> rm -rf /home/ubuntu/cephtest/cephadm 2024-02-07T12:44:05.831 DEBUG:teuthology.orchestra.run.smithi184:> rm -rf /home/ubuntu/cephtest/cephadm 2024-02-07T12:44:05.846 INFO:tasks.cephadm:Teardown complete 2024-02-07T12:44:05.847 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-02-07T12:44:05.858 INFO:teuthology.task.clock:Checking final clock skew... 2024-02-07T12:44:05.859 DEBUG:teuthology.orchestra.run.smithi174:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-07T12:44:05.874 DEBUG:teuthology.orchestra.run.smithi184:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-02-07T12:44:05.885 INFO:teuthology.orchestra.run.smithi174.stderr:bash: line 1: ntpq: command not found 2024-02-07T12:44:05.890 INFO:teuthology.orchestra.run.smithi174.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-07T12:44:05.890 INFO:teuthology.orchestra.run.smithi174.stdout:=============================================================================== 2024-02-07T12:44:05.890 INFO:teuthology.orchestra.run.smithi174.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 6 -763us[ -763us] +/- 90ms 2024-02-07T12:44:05.890 INFO:teuthology.orchestra.run.smithi174.stdout:^* hv02.front.sepia.ceph.com 3 6 377 11 -87us[ -58us] +/- 59ms 2024-02-07T12:44:05.890 INFO:teuthology.orchestra.run.smithi174.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 12 +823us[ +851us] +/- 77ms 2024-02-07T12:44:05.890 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:44:05.899 INFO:teuthology.orchestra.run.smithi184.stderr:bash: line 1: ntpq: command not found 2024-02-07T12:44:05.903 INFO:teuthology.orchestra.run.smithi184.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-02-07T12:44:05.904 INFO:teuthology.orchestra.run.smithi184.stdout:=============================================================================== 2024-02-07T12:44:05.904 INFO:teuthology.orchestra.run.smithi184.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 6 -765us[ -765us] +/- 90ms 2024-02-07T12:44:05.904 INFO:teuthology.orchestra.run.smithi184.stdout:^* hv02.front.sepia.ceph.com 3 6 377 9 -92us[ -77us] +/- 59ms 2024-02-07T12:44:05.904 INFO:teuthology.orchestra.run.smithi184.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 9 +815us[ +815us] +/- 77ms 2024-02-07T12:44:05.904 INFO:teuthology.orchestra.run.smithi184.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-02-07T12:44:05.904 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-02-07T12:44:05.915 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-02-07T12:44:05.917 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-02-07T12:44:05.936 DEBUG:teuthology.orchestra.run.smithi174:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-02-07T12:44:06.018 DEBUG:teuthology.orchestra.run.smithi184:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-02-07T12:44:06.095 DEBUG:teuthology.orchestra.run.smithi174:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-07T12:44:06.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:44:06.126 DEBUG:teuthology.orchestra.run.smithi184:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-02-07T12:44:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:44:06.156 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-02-07T12:44:06.167 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-02-07T12:44:06.177 INFO:teuthology.task.internal:Duration was 2076.932551 seconds 2024-02-07T12:44:06.177 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-02-07T12:44:06.193 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-02-07T12:44:06.193 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-02-07T12:44:06.195 DEBUG:teuthology.orchestra.run.smithi184:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-02-07T12:44:06.237 INFO:teuthology.orchestra.run.smithi174.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-07T12:44:06.239 INFO:teuthology.orchestra.run.smithi184.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-02-07T12:44:06.579 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-02-07T12:44:06.579 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi174.front.sepia.ceph.com 2024-02-07T12:44:06.580 DEBUG:teuthology.orchestra.run.smithi174:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-02-07T12:44:06.639 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi184.front.sepia.ceph.com 2024-02-07T12:44:06.639 DEBUG:teuthology.orchestra.run.smithi184:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-02-07T12:44:06.662 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-02-07T12:44:06.663 DEBUG:teuthology.orchestra.run.smithi174:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-02-07T12:44:06.681 DEBUG:teuthology.orchestra.run.smithi184:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-02-07T12:44:06.731 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-02-07T12:44:06.731 DEBUG:teuthology.orchestra.run.smithi174:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-02-07T12:44:06.749 DEBUG:teuthology.orchestra.run.smithi184:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-02-07T12:44:20.764 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-02-07T12:44:20.777 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-02-07T12:44:20.777 DEBUG:teuthology.orchestra.run.smithi174:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-02-07T12:44:20.804 DEBUG:teuthology.orchestra.run.smithi184:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-02-07T12:44:20.837 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-02-07T12:44:20.848 DEBUG:teuthology.orchestra.run.smithi174:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-02-07T12:44:20.851 DEBUG:teuthology.orchestra.run.smithi184:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-02-07T12:44:20.873 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern = core 2024-02-07T12:44:20.907 INFO:teuthology.orchestra.run.smithi184.stdout:kernel.core_pattern = core 2024-02-07T12:44:20.942 DEBUG:teuthology.orchestra.run.smithi174:> test -e /home/ubuntu/cephtest/archive/coredump 2024-02-07T12:44:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:44:20.982 DEBUG:teuthology.orchestra.run.smithi184:> test -e /home/ubuntu/cephtest/archive/coredump 2024-02-07T12:44:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-02-07T12:44:21.000 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-02-07T12:44:21.010 INFO:teuthology.task.internal:Transferring archived files... 2024-02-07T12:44:21.011 DEBUG:teuthology.misc:Transferring archived files from smithi174:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725/remote/smithi174 2024-02-07T12:44:21.011 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-02-07T12:44:21.086 DEBUG:teuthology.misc:Transferring archived files from smithi184:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725/remote/smithi184 2024-02-07T12:44:21.086 DEBUG:teuthology.orchestra.run.smithi184:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-02-07T12:44:21.133 INFO:teuthology.task.internal:Removing archive directory... 2024-02-07T12:44:21.133 DEBUG:teuthology.orchestra.run.smithi174:> rm -rf -- /home/ubuntu/cephtest/archive 2024-02-07T12:44:21.135 DEBUG:teuthology.orchestra.run.smithi184:> rm -rf -- /home/ubuntu/cephtest/archive 2024-02-07T12:44:21.183 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-02-07T12:44:21.197 INFO:teuthology.task.internal:Not uploading archives. 2024-02-07T12:44:21.197 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-02-07T12:44:21.209 INFO:teuthology.task.internal:Tidying up after the test... 2024-02-07T12:44:21.209 DEBUG:teuthology.orchestra.run.smithi174:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-02-07T12:44:21.211 DEBUG:teuthology.orchestra.run.smithi184:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-02-07T12:44:21.223 INFO:teuthology.orchestra.run.smithi174.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 7 12:44 /home/ubuntu/cephtest 2024-02-07T12:44:21.239 INFO:teuthology.orchestra.run.smithi184.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 7 12:44 /home/ubuntu/cephtest 2024-02-07T12:44:21.240 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-02-07T12:44:21.252 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-02-07T12:44:21.320 INFO:teuthology.nuke:Checking targets against current locks 2024-02-07T12:44:21.344 DEBUG:teuthology.nuke:shortname: smithi174 2024-02-07T12:44:21.344 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-07T12:44:21.363 DEBUG:teuthology.nuke:shortname: smithi184 2024-02-07T12:44:21.363 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-02-07T12:44:21.366 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi174.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-07 12:00:52.249965', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHGHDzVTMmvRCe98+h4WjUrmQsWynO+p1JNnVlkSvZSdLvFqEUqU9ip9vRdvep7DLjAPaCcdN8d7jdl/0UMK7Qw='} 2024-02-07T12:44:21.386 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi184.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-02-06_08:19:47-orch:cephadm-wip-guits-testing-4-2024-02-05-1854-distro-default-smithi/7548725', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-02-07 12:00:52.251928', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjya6K0fUqaU5b91JgaID3NERPMzvfIW0l2bJaoxsbz9RvWVhoAa6Dr2icZzARkaQ30DsvaY/siZxF6JxrsA4A='} 2024-02-07T12:44:21.414 INFO:teuthology.orchestra.console.smithi174:Power off 2024-02-07T12:44:21.414 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-02-07T12:44:21.442 DEBUG:teuthology.orchestra.console.smithi174:power off output: Chassis Power Control: Down/Off 2024-02-07T12:44:21.442 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T12:44:21.458 INFO:teuthology.orchestra.console.smithi184:Power off 2024-02-07T12:44:21.458 DEBUG:teuthology.orchestra.console.smithi184:pexpect command: ipmitool -H smithi184.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-02-07T12:44:21.468 DEBUG:teuthology.orchestra.console.smithi174:check power output: Chassis Power is on 2024-02-07T12:44:21.481 DEBUG:teuthology.orchestra.console.smithi184:power off output: Chassis Power Control: Down/Off 2024-02-07T12:44:21.481 DEBUG:teuthology.orchestra.console.smithi184:pexpect command: ipmitool -H smithi184.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T12:44:21.506 DEBUG:teuthology.orchestra.console.smithi184:check power output: Chassis Power is on 2024-02-07T12:44:25.469 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T12:44:25.506 DEBUG:teuthology.orchestra.console.smithi184:pexpect command: ipmitool -H smithi184.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T12:44:25.581 DEBUG:teuthology.orchestra.console.smithi174:check power output: Chassis Power is on 2024-02-07T12:44:25.616 DEBUG:teuthology.orchestra.console.smithi184:check power output: Chassis Power is on 2024-02-07T12:44:29.584 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T12:44:29.617 DEBUG:teuthology.orchestra.console.smithi184:pexpect command: ipmitool -H smithi184.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-02-07T12:44:29.694 DEBUG:teuthology.orchestra.console.smithi174:check power output: Chassis Power is off 2024-02-07T12:44:29.731 DEBUG:teuthology.orchestra.console.smithi184:check power output: Chassis Power is off 2024-02-07T12:44:29.796 INFO:teuthology.orchestra.console.smithi174:Power off completed 2024-02-07T12:44:29.832 INFO:teuthology.orchestra.console.smithi184:Power off completed 2024-02-07T12:44:29.937 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2076.9325506687164 failure_reason: '"2024-02-07T12:19:28.182128+0000 mon.smithi174 (mon.0) 491 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log' owner: scheduled_gabrioux@teuthology success: false 2024-02-07T12:44:29.937 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-02-07T12:44:30.017 INFO:teuthology.run:FAIL